diff --git a/TODO.org b/TODO.org index ef1d737..209816f 100644 --- a/TODO.org +++ b/TODO.org @@ -1,9 +1,14 @@ #+title: backlog #+author: ardumont -* TODO 0.0.3 [%] -- [ ] Deliver to melpa -- [ ] Deliver to el-get +* DONE 0.0.3 [100%] +CLOSED: [2015-08-08 Sat 20:56] +- [X] Simplify json parsing +- [X] Fix total tasks done problems when multiple tasks +- [X] Add group for customization group +- [X] melpa - https://github.com/milkypostman/melpa/pull/3002 +- [X] el-get - https://github.com/dimitri/el-get/pull/2230 +- [X] Release notes * DONE 0.0.2 [100%] CLOSED: [2015-08-08 Sat 11:33] diff --git a/celery.el b/celery.el index 7cd5492..5ad3599 100644 --- a/celery.el +++ b/celery.el @@ -47,15 +47,22 @@ (require 'json) (require 's) +(defgroup celery nil " Celery customisation group." + :tag "Celery" + :version "0.0.3" + :group 'celery) + (defcustom celery-command "celery" "The celery command in charge of outputing the result this mode parse. The user can override this. For example, if a remote machine only knows celery, it could be defined as: -\(custom-set-variables '\(celery-command \"ssh remote-node celery\"\)\)") +\(custom-set-variables '\(celery-command \"ssh remote-node celery\"\)\)" + :group 'celery) (defcustom celery-workers-list nil "If non nil, filter the stats according to the content of this list. -This is a list of worker names.") +This is a list of worker names." + :group 'celery) (defvar celery-last-known-stats nil "Latest worker stats. @@ -84,11 +91,7 @@ Mostly to work offline.") (defun celery-compute-full-stats-workers () "Compute the worker' stats in json data structure." - (let ((stats-json-str-output (celery--compute-json-string-stats))) - (with-temp-buffer - (insert stats-json-str-output) - (goto-char (point-min)) - (json-read)))) + (json-read-from-string (celery--compute-json-string-stats))) (defun celery-total-tasks-per-worker (stats worker) "Compute the total number of tasks from STATS for WORKER." @@ -139,19 +142,19 @@ Mostly to work offline.") (defun celery-full-stats-count-processes-per-worker (full-stats worker) "Access processes stats from FULL-STATS for the WORKER." - (-when-let (w (assoc-default worker full-stats)) - (->> w + (-when-let (worker-stats (assoc-default worker full-stats)) + (->> worker-stats (assoc-default 'pool) (assoc-default 'processes) length))) (defun celery-full-stats-total-tasks-per-worker (full-stats worker) "Compute the total number of tasks from FULL-STATS for WORKER." - (-when-let (w (assoc-default worker full-stats)) - (->> w + (-when-let (worker-stats (assoc-default worker full-stats)) + (->> worker-stats (assoc-default 'total) - car - cdr))) + (mapcar #'cdr) + (apply #'+)))) (defun celery-simplify-stats (full-stats) "Compute the number of total tasks done per worker from the FULL-STATS." @@ -201,13 +204,15 @@ So this needs to be applied in an org context to make sense." (interactive "P") (celery--with-delay-apply 'celery--stats-to-org-row refresh)) +(defalias 'celery-log-stats (-partial 'celery-log "Stats: %s")) + ;;;###autoload (defun celery-compute-stats-workers (&optional refresh) "Compute the simplified workers' stats. if REFRESH is non nil, trigger a computation. Otherwise, reuse the latest known values." (interactive "P") - (celery--with-delay-apply (-partial 'celery-log "Stats: %s") refresh)) + (celery--with-delay-apply 'celery-log-stats refresh)) (defun celery-all-tasks-consumed (stats) "Compute the total number of consumed tasks from the STATS." @@ -215,15 +220,16 @@ Otherwise, reuse the latest known values." (mapcar (-partial 'assoc-default :total)) (apply #'+))) +(defalias 'celery-log-total-tasks-consumed + (-compose (-partial 'celery-log "Number of total tasks done: %s") + 'celery-all-tasks-consumed)) + ;;;###autoload (defun celery-compute-tasks-consumed-workers (&optional refresh) "Check the current number of tasks executed by workers in celery. if REFRESH is mentioned, trigger a check, otherwise, use the latest value." (interactive "P") - (celery--with-delay-apply - (-compose (-partial 'celery-log "Number of total tasks done: %s") - 'celery-all-tasks-consumed) - refresh)) + (celery--with-delay-apply 'celery-log-total-tasks-consumed refresh)) (defvar celery-mode-map (let ((map (make-sparse-keymap))) @@ -235,12 +241,6 @@ if REFRESH is mentioned, trigger a check, otherwise, use the latest value." ;;;###autoload (define-minor-mode celery-mode - - - - - - "Minor mode to consolidate Emacs' celery extensions. \\{celery-mode-map}" diff --git a/release-notes.md b/release-notes.md index 79d6a24..6a8fc8c 100644 --- a/release-notes.md +++ b/release-notes.md @@ -1,3 +1,12 @@ +# 0.0.3 + +- [X] Simplify json parsing +- [X] Fix total tasks done problems when multiple tasks +- [X] Add group for customization group +- [X] melpa - https://github.com/milkypostman/melpa/pull/3002 +- [X] el-get - https://github.com/dimitri/el-get/pull/2230 +- [X] Release notes + # 0.0.2 - [X] Fix release script error diff --git a/test/celery-test.el b/test/celery-test.el index e3b6b9c..fa3c31a 100644 --- a/test/celery-test.el +++ b/test/celery-test.el @@ -3,9 +3,81 @@ (require 'celery) +(ert-deftest test-celery--compute-json-string-stats () + (should (string= "{ + \"worker02\": + { + \"broker\": { + \"alternates\": [], + \"connect_timeout\": 4, + \"heartbeat\": null, + \"hostname\": \"moma\", + \"insist\": false, + \"login_method\": \"AMQPLAIN\", + \"port\": 5672, + \"ssl\": false, + \"transport\": \"amqp\", + \"transport_options\": {}, + \"uri_prefix\": null, + \"userid\": \"guest\", + \"virtual_host\": \"/\" + }, + \"clock\": \"403897\", + \"pid\": 14216, + \"pool\": { + \"max-concurrency\": 1, + \"max-tasks-per-child\": \"N/A\", + \"processes\": [ + 14221, + 14223 + ], + \"put-guarded-by-semaphore\": false, + \"timeouts\": [ + 3600.0, + 0 + ], + \"writes\": { + \"all\": \"46.85%, 53.15%\", + \"avg\": \"50.00%\", + \"inqueues\": { + \"active\": 0, + \"total\": 2 + }, + \"raw\": \"12702, 14411\", + \"total\": 27113 + } + }, + \"prefetch_count\": 8, + \"rusage\": { + \"idrss\": 0, + \"inblock\": 14280, + \"isrss\": 0, + \"ixrss\": 0, + \"majflt\": 38, + \"maxrss\": 36568, + \"minflt\": 15180, + \"msgrcv\": 0, + \"msgsnd\": 0, + \"nivcsw\": 132094, + \"nsignals\": 0, + \"nswap\": 0, + \"nvcsw\": 208050, + \"oublock\": 0, + \"stime\": 26.512, + \"utime\": 393.488 + }, + \"total\": { + \"worker.tasks.do_something_awesome\": 27113 + } + } +}" + (with-mock + (mock (celery--compute-raw-celery-output) => "celery@worker02: OK\n {\n \"broker\": {\n \"alternates\": [],\n \"connect_timeout\": 4,\n \"heartbeat\": null,\n \"hostname\": \"moma\",\n \"insist\": false,\n \"login_method\": \"AMQPLAIN\",\n \"port\": 5672,\n \"ssl\": false,\n \"transport\": \"amqp\",\n \"transport_options\": {},\n \"uri_prefix\": null,\n \"userid\": \"guest\",\n \"virtual_host\": \"/\"\n },\n \"clock\": \"403897\",\n \"pid\": 14216,\n \"pool\": {\n \"max-concurrency\": 1,\n \"max-tasks-per-child\": \"N/A\",\n \"processes\": [\n 14221,\n 14223\n ],\n \"put-guarded-by-semaphore\": false,\n \"timeouts\": [\n 3600.0,\n 0\n ],\n \"writes\": {\n \"all\": \"46.85%, 53.15%\",\n \"avg\": \"50.00%\",\n \"inqueues\": {\n \"active\": 0,\n \"total\": 2\n },\n \"raw\": \"12702, 14411\",\n \"total\": 27113\n }\n },\n \"prefetch_count\": 8,\n \"rusage\": {\n \"idrss\": 0,\n \"inblock\": 14280,\n \"isrss\": 0,\n \"ixrss\": 0,\n \"majflt\": 38,\n \"maxrss\": 36568,\n \"minflt\": 15180,\n \"msgrcv\": 0,\n \"msgsnd\": 0,\n \"nivcsw\": 132094,\n \"nsignals\": 0,\n \"nswap\": 0,\n \"nvcsw\": 208050,\n \"oublock\": 0,\n \"stime\": 26.512,\n \"utime\": 393.488\n },\n \"total\": {\n \"worker.tasks.do_something_awesome\": 27113\n }\n }") + (celery--compute-json-string-stats))))) + (ert-deftest test-celery-compute-full-stats-workers () (should (equal '((worker02 (total - (swh\.worker\.tasks\.do_something_awesome . 27113)) + (worker\.tasks\.do_something_awesome . 27113)) (rusage (utime . 393.488) (stime . 26.512) @@ -59,7 +131,7 @@ [])))) (with-mock (mock (celery--compute-raw-celery-output) => - "celery@worker02: OK\n {\n \"broker\": {\n \"alternates\": [],\n \"connect_timeout\": 4,\n \"heartbeat\": null,\n \"hostname\": \"moma\",\n \"insist\": false,\n \"login_method\": \"AMQPLAIN\",\n \"port\": 5672,\n \"ssl\": false,\n \"transport\": \"amqp\",\n \"transport_options\": {},\n \"uri_prefix\": null,\n \"userid\": \"guest\",\n \"virtual_host\": \"/\"\n },\n \"clock\": \"403897\",\n \"pid\": 14216,\n \"pool\": {\n \"max-concurrency\": 1,\n \"max-tasks-per-child\": \"N/A\",\n \"processes\": [\n 14221,\n 14223\n ],\n \"put-guarded-by-semaphore\": false,\n \"timeouts\": [\n 3600.0,\n 0\n ],\n \"writes\": {\n \"all\": \"46.85%, 53.15%\",\n \"avg\": \"50.00%\",\n \"inqueues\": {\n \"active\": 0,\n \"total\": 2\n },\n \"raw\": \"12702, 14411\",\n \"total\": 27113\n }\n },\n \"prefetch_count\": 8,\n \"rusage\": {\n \"idrss\": 0,\n \"inblock\": 14280,\n \"isrss\": 0,\n \"ixrss\": 0,\n \"majflt\": 38,\n \"maxrss\": 36568,\n \"minflt\": 15180,\n \"msgrcv\": 0,\n \"msgsnd\": 0,\n \"nivcsw\": 132094,\n \"nsignals\": 0,\n \"nswap\": 0,\n \"nvcsw\": 208050,\n \"oublock\": 0,\n \"stime\": 26.512,\n \"utime\": 393.488\n },\n \"total\": {\n \"swh.worker.tasks.do_something_awesome\": 27113\n }\n }") + "celery@worker02: OK\n {\n \"broker\": {\n \"alternates\": [],\n \"connect_timeout\": 4,\n \"heartbeat\": null,\n \"hostname\": \"moma\",\n \"insist\": false,\n \"login_method\": \"AMQPLAIN\",\n \"port\": 5672,\n \"ssl\": false,\n \"transport\": \"amqp\",\n \"transport_options\": {},\n \"uri_prefix\": null,\n \"userid\": \"guest\",\n \"virtual_host\": \"/\"\n },\n \"clock\": \"403897\",\n \"pid\": 14216,\n \"pool\": {\n \"max-concurrency\": 1,\n \"max-tasks-per-child\": \"N/A\",\n \"processes\": [\n 14221,\n 14223\n ],\n \"put-guarded-by-semaphore\": false,\n \"timeouts\": [\n 3600.0,\n 0\n ],\n \"writes\": {\n \"all\": \"46.85%, 53.15%\",\n \"avg\": \"50.00%\",\n \"inqueues\": {\n \"active\": 0,\n \"total\": 2\n },\n \"raw\": \"12702, 14411\",\n \"total\": 27113\n }\n },\n \"prefetch_count\": 8,\n \"rusage\": {\n \"idrss\": 0,\n \"inblock\": 14280,\n \"isrss\": 0,\n \"ixrss\": 0,\n \"majflt\": 38,\n \"maxrss\": 36568,\n \"minflt\": 15180,\n \"msgrcv\": 0,\n \"msgsnd\": 0,\n \"nivcsw\": 132094,\n \"nsignals\": 0,\n \"nswap\": 0,\n \"nvcsw\": 208050,\n \"oublock\": 0,\n \"stime\": 26.512,\n \"utime\": 393.488\n },\n \"total\": {\n \"worker.tasks.do_something_awesome\": 27113\n }\n }") (celery-compute-full-stats-workers))))) @@ -79,7 +151,7 @@ (should-not (let ((stats '((w01 (:total 27113))))) (celery-total-tasks-per-worker stats 'w02)))) -(ert-deftest test-celery-total-tasks-per-worker () +(ert-deftest test-celery--to-org-table-row () (should (string= "| Fri Aug 7 19:02:12 2015 | 27113 | 300 | " (with-mock (mock (current-time-string) => "Fri Aug 7 19:02:12 2015") @@ -184,16 +256,6 @@ (let ((celery-last-known-stats :old-stats)) (celery--compute-stats-workers-with-refresh 'with-refresh)))))) -;; (ert-deftest test-celery--with-delay-apply () -;; (with-mock -;; (mock (celery--compute-stats-workers-with-refresh) => :stats) -;; (mock (celery-simplify-stats :stats) => :simplified-stats) -;; (celery--with-delay-apply -;; (lambda (stats) -;; (message "stats: %s" stats) -;; (should (equal :simplified-stats stats)))))) -;; does not work - (ert-deftest test-celery-all-tasks-consumed () (should (equal 6000 (celery-all-tasks-consumed '((worker02 (:total . 3600) (:processes . 2)) @@ -221,9 +283,11 @@ (celery-full-stats-count-processes-per-worker stats 'w02)))) (ert-deftest test-celery-full-stats-total-tasks-per-worker () - (should (equal 27113 (celery-full-stats-total-tasks-per-worker + (should (equal 30000 (celery-full-stats-total-tasks-per-worker '((w01 (total - (swh\.worker\.tasks\.do_something_awesome . 27113)))) + (do_something_awesome . 27000) + (do_awesomer_stuff . 2000) + (yet_another_task . 1000)))) 'w01))) (should-not (celery-full-stats-total-tasks-per-worker '((w01 (total @@ -236,3 +300,33 @@ (with-mock (mock (shell-command-to-string "celery inspect stats --quiet --no-color") => :foobar) (celery--compute-raw-celery-output)))))) + +(ert-deftest test-celery-log-total-tasks-consumed () + (should (string= "Celery - Number of total tasks done: 6000" + (celery-log-total-tasks-consumed '((w01 (:total . 1000) (:processes . 1)) + (w02 (:total . 2000) (:processes . 2)) + (w03 (:total . 3000) (:processes . 3))))))) + +(ert-deftest test-celery-log-stats () + (should (string= "Celery - Stats: ((w01 (:total . 1000) (:processes . 1)) (w02 (:total . 2000) (:processes . 2)) (w03 (:total . 3000) (:processes . 3)))" + (celery-log-stats '((w01 (:total . 1000) (:processes . 1)) + (w02 (:total . 2000) (:processes . 2)) + (w03 (:total . 3000) (:processes . 3))))))) + +(ert-deftest test-celery-stats-to-org-row () + (should (eq :res + (with-mock + (mock (celery--with-delay-apply 'celery--stats-to-org-row :refresh) => :res) + (celery-stats-to-org-row :refresh))))) + +(ert-deftest test-celery-compute-tasks-consumed-workers () + (should (eq :res + (with-mock + (mock (celery--with-delay-apply 'celery-log-total-tasks-consumed :refresh) => :res) + (celery-compute-tasks-consumed-workers :refresh))))) + +(ert-deftest test-celery-compute-stats-workers () + (should (eq :res + (with-mock + (mock (celery--with-delay-apply 'celery-log-stats :refresh) => :res) + (celery-compute-stats-workers :refresh)))))