From 25f399b08e925827d1fd1536e0d40604a91e6ed8 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 16 Jun 2017 10:30:51 -0600 Subject: [PATCH 01/10] adding timer --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f807e2fe..d733d36cd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: # install a few of the dependencies that pip would otherwise try to install # when intalling scikit-bio - travis_retry conda create --yes -n qiita python=$PYTHON_VERSION pip nose flake8 - pyzmq networkx pyparsing natsort mock future libgfortran seaborn + pyzmq networkx pyparsing natsort mock future libgfortran seaborn nose-timer 'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1' - source activate qiita - pip install -U pip @@ -56,7 +56,7 @@ script: - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi - - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v --cover-package=qiita_db,qiita_pet,qiita_core,qiita_ware; fi + - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage --with-timer -v --cover-package=qiita_db,qiita_pet,qiita_core,qiita_ware; fi - flake8 qiita_* setup.py scripts/* - ls -R /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_pet/support_files/doc/ - qiita pet webserver From 4d95a80db34636d810795a841fe32f19ee638557 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 16 Jun 2017 10:38:23 -0600 Subject: [PATCH 02/10] nose-timer to pip command --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d733d36cd..65d0d7086 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,11 +23,11 @@ install: # install a few of the dependencies that pip would otherwise try to install # when intalling scikit-bio - travis_retry conda create --yes -n qiita python=$PYTHON_VERSION pip nose flake8 - pyzmq networkx pyparsing natsort mock future libgfortran seaborn nose-timer + pyzmq networkx pyparsing natsort mock future libgfortran seaborn 'pandas>=0.18' 'matplotlib>=1.1.0' 'scipy>0.13.0' 'numpy>=1.7' 'h5py>=2.3.1' - source activate qiita - pip install -U pip - - pip install sphinx sphinx-bootstrap-theme coveralls 'ipython[all]==2.4.1' + - pip install sphinx sphinx-bootstrap-theme coveralls 'ipython[all]==2.4.1' nose-timer - travis_retry pip install . --process-dependency-links - 'echo "backend: Agg" > matplotlibrc' # Install the biom plugin so we can run the analysis tests From 47d54df09d9c2986fe672316e8aaec2b8de38d09 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 16 Jun 2017 15:07:32 -0600 Subject: [PATCH 03/10] active_children test --- scripts/qiita | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/qiita b/scripts/qiita index ac0053413..3670ff171 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -16,7 +16,7 @@ from os.path import join, abspath, dirname, basename from future.utils import viewitems from glob import glob from time import sleep -from multiprocessing import active_children +# from multiprocessing import active_children import click import tornado.httpserver @@ -464,7 +464,7 @@ def start(port, master): # Set a PeriodicCallback for cleaning up the threads every 10 seconds # To understand why this is working as expected, check the multiprocessing # documentation https://docs.python.org/2/library/multiprocessing.html - PeriodicCallback(lambda: active_children(), 10000).start() + # PeriodicCallback(lambda: active_children(), 10000).start() ioloop.start() From 4a5f89861b206a0ecf47c7446fbac131eb9c9bfb Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Sun, 18 Jun 2017 09:53:00 -0600 Subject: [PATCH 04/10] 300000 --- scripts/qiita | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qiita b/scripts/qiita index 3670ff171..c9b99bcad 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -464,7 +464,7 @@ def start(port, master): # Set a PeriodicCallback for cleaning up the threads every 10 seconds # To understand why this is working as expected, check the multiprocessing # documentation https://docs.python.org/2/library/multiprocessing.html - # PeriodicCallback(lambda: active_children(), 10000).start() + PeriodicCallback(lambda: active_children(), 300000).start() ioloop.start() From 9eff24a8bc4af49cb4c573f9c2d020d22ccb48f6 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 19 Jun 2017 06:24:00 -0600 Subject: [PATCH 05/10] multiprocessing->active_children --- scripts/qiita | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qiita b/scripts/qiita index c9b99bcad..5876d5d14 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -16,7 +16,7 @@ from os.path import join, abspath, dirname, basename from future.utils import viewitems from glob import glob from time import sleep -# from multiprocessing import active_children +from multiprocessing import active_children import click import tornado.httpserver From c245c04dd1bef20b114045228031f05e517cdc7e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 19 Jun 2017 07:14:43 -0600 Subject: [PATCH 06/10] 2400000 --- scripts/qiita | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/qiita b/scripts/qiita index 5876d5d14..22ffa0b03 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -464,7 +464,8 @@ def start(port, master): # Set a PeriodicCallback for cleaning up the threads every 10 seconds # To understand why this is working as expected, check the multiprocessing # documentation https://docs.python.org/2/library/multiprocessing.html - PeriodicCallback(lambda: active_children(), 300000).start() + # 2400000 == 40 min + PeriodicCallback(lambda: active_children(), 2400000).start() ioloop.start() From 82751f6b672bc7b2daf1a78fc1378582fa724982 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Mon, 19 Jun 2017 08:08:55 -0600 Subject: [PATCH 07/10] 600000 --- scripts/qiita | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/qiita b/scripts/qiita index 22ffa0b03..65153b0ba 100755 --- a/scripts/qiita +++ b/scripts/qiita @@ -464,8 +464,8 @@ def start(port, master): # Set a PeriodicCallback for cleaning up the threads every 10 seconds # To understand why this is working as expected, check the multiprocessing # documentation https://docs.python.org/2/library/multiprocessing.html - # 2400000 == 40 min - PeriodicCallback(lambda: active_children(), 2400000).start() + # 600000 == 10 min + PeriodicCallback(lambda: active_children(), 600000).start() ioloop.start() From c2086b7272fdf7053b630220f6e4523ee927d22f Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Mon, 19 Jun 2017 08:27:36 -0700 Subject: [PATCH 08/10] Trying to fix the tests --- qiita_db/analysis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index 048d1f5ce..2421322c4 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -183,8 +183,10 @@ def create(cls, owner, name, description, from_default=False, VALUES (%s, %s)""" qdb.sql_connection.TRN.add(sql, [a_id, job.id]) qdb.sql_connection.TRN.execute() - job.submit() - return instance + + # Doing the submission outside of the transaction + job.submit() + return instance @classmethod def delete(cls, _id): From 4b17051cc6b709c8c8e514e87b137f2be5b5eca2 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Mon, 19 Jun 2017 09:28:44 -0700 Subject: [PATCH 09/10] Submit the job on commit --- qiita_db/analysis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index 2421322c4..e54e785fc 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -184,8 +184,9 @@ def create(cls, owner, name, description, from_default=False, qdb.sql_connection.TRN.add(sql, [a_id, job.id]) qdb.sql_connection.TRN.execute() - # Doing the submission outside of the transaction - job.submit() + # Don't submit the job until the transaction commits + qdb.sql_connection.TRN.add_post_commit_func(job.submit) + return instance @classmethod From cec7ca448405358814405eef2ce5cf8c04a0bba3 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Mon, 19 Jun 2017 09:42:40 -0700 Subject: [PATCH 10/10] Reverting last change --- qiita_db/analysis.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index e54e785fc..2421322c4 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -184,9 +184,8 @@ def create(cls, owner, name, description, from_default=False, qdb.sql_connection.TRN.add(sql, [a_id, job.id]) qdb.sql_connection.TRN.execute() - # Don't submit the job until the transaction commits - qdb.sql_connection.TRN.add_post_commit_func(job.submit) - + # Doing the submission outside of the transaction + job.submit() return instance @classmethod