From 0650a515a6abf6ae4440560dcb16207c9cf0902e Mon Sep 17 00:00:00 2001 From: ErinWeisbart <54687786+ErinWeisbart@users.noreply.github.com> Date: Tue, 9 Jul 2024 15:04:48 -0700 Subject: [PATCH 1/4] update cellprofiler-plugins organization --- worker/cp-worker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/worker/cp-worker.py b/worker/cp-worker.py index b2a1c74..f33f6eb 100644 --- a/worker/cp-worker.py +++ b/worker/cp-worker.py @@ -12,7 +12,7 @@ DATA_ROOT = '/home/ubuntu/bucket' LOCAL_OUTPUT = '/home/ubuntu/local_output' -PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins' +PLUGIN_DIR = '/home/ubuntu/CellProfiler-plugins/active_plugins' QUEUE_URL = os.environ['SQS_QUEUE_URL'] AWS_BUCKET = os.environ['AWS_BUCKET'] if 'SOURCE_BUCKET' not in os.environ: @@ -275,7 +275,11 @@ def runCellProfiler(message): else: printandlog("Didn't recognize input file",logger) if USE_PLUGINS.lower() == 'true': - cmd += f' --plugins-directory={PLUGIN_DIR}' + if os.path.isdir(PLUGIN_DIR): + cmd += f' --plugins-directory={PLUGIN_DIR}' + else: + printandlog("Can't find current 'active_plugins' folder, defaulting to plugins root directory.",logger) + cmd += " --plugins-directory='/home/ubuntu/CellProfiler-plugins/'" print(f'Running {cmd}') logger.info(cmd) From 0d0794c4e5c1fe0625a430f5ca821c656f3631e5 Mon Sep 17 00:00:00 2001 From: ErinWeisbart <54687786+ErinWeisbart@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:52:45 -0700 Subject: [PATCH 2/4] more granular,updated plugins options --- config.py | 10 +++---- .../DCP-documentation/step_1_configuration.md | 12 ++++++-- run.py | 7 +++-- worker/run-worker.sh | 30 +++++++++++++++---- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 6840a7f..ad8e8a1 100644 --- a/config.py +++ b/config.py @@ -50,8 +50,8 @@ NECESSARY_STRING = '' #Is there any string that should be in the file name to "count"? # PLUGINS -USE_PLUGINS = 'False' # True to use any plugin from CellProfiler-plugins repo -UPDATE_PLUGINS = 'False' # True to download updates from CellProfiler-plugins repo -PLUGINS_COMMIT = '' # What commit or version tag do you want to check out? -INSTALL_REQUIREMENTS = 'False' # True to install REQUIREMENTS_FILE defined below. Requirements should have all plugin dependencies. -REQUIREMENTS_FILE = '' # Path within the CellProfiler-plugins repo to a requirements file +USE_PLUGINS = 'False' # True to use any plugin from CellProfiler-plugins repo +UPDATE_PLUGINS = 'False' # True to download updates from CellProfiler-plugins repo +PLUGINS_COMMIT = 'False' # What commit or version tag do you want to check out? If not, set to False. +INSTALL_REQUIREMENTS = 'False' # True to install REQUIREMENTS defined below. Requirements should have all plugin dependencies. +REQUIREMENTS = '' # Flag to use with install (current) or path within the CellProfiler-plugins repo to a requirements file (deprecated). diff --git a/documentation/DCP-documentation/step_1_configuration.md b/documentation/DCP-documentation/step_1_configuration.md index 272cbe7..5100dd2 100644 --- a/documentation/DCP-documentation/step_1_configuration.md +++ b/documentation/DCP-documentation/step_1_configuration.md @@ -111,13 +111,19 @@ Useful when trying to detect jobs that may have exported smaller corrupted files ### PLUGINS * **USE_PLUGINS:** Whether or not you will be using external plugins from the CellProfiler-plugins repository. +When True, passes the `--plugins-directory` flag to CellProfiler. +Defaults to the current v1.0 `CellProfiler-plugins/active_plugins` location for plugins but will revert to the historical location of plugins in the `CellProfiler-plugins` root directory if the `active_plugins` folder is not present. * **UPDATE_PLUGINS:** Whether or not to update the plugins repository before use. (i.e. run `git fetch --all` on CellProfiler-plugins) -* **PLUGINS_COMMIT:** What commit or version tag to check out. +* **PLUGINS_COMMIT:** If desired, what commit or version tag to check out. Used in the `git checkout PLUGINS_COMMIT` command in CellProfiler-plugins. +If you do not want to checkout a specific commit, set to False. * **INSTALL_REQUIREMENTS:** Whether or not to install requirements associate with plugins. -* **REQUIREMENTS_FILE:** A path within the CellProfiler-plugins repository to the requirements file you would like to install. -Used in the `pip install -r REQUIREMENTS_FILE` command. +Not all plugins require additional requirement installation. +See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on requirements. +* **REQUIREMENTS:** For current v1.0 CellProfiler-plugins, a flag that will be passed to the install command (e.g. `cellpose`). +See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on supported flags. +For deprecated versions of CellProfiler-plugins before v1.0, pass a path within the CellProfiler-plugins repository to the requirements file you would like to install that will be used in the `pip install -r REQUIREMENTS_FILE` command. *** diff --git a/run.py b/run.py index 6ef6a05..4cbdab5 100644 --- a/run.py +++ b/run.py @@ -17,12 +17,15 @@ CREATE_DASHBOARD = 'False' CLEAN_DASHBOARD = 'False' AUTO_MONITOR = 'False' +REQUIREMENTS_FILE = 'False' from config import * # Back compatability with old config requirements if ':' in SQS_DEAD_LETTER_QUEUE: SQS_DEAD_LETTER_QUEUE = SQS_DEAD_LETTER_QUEUE.rsplit(':',1)[1] +if REQUIREMENTS_FILE: + REQUIREMENTS = REQUIREMENTS_FILE WAIT_TIME = 60 MONITOR_TIME = 60 @@ -147,12 +150,12 @@ def generate_task_definition(AWS_PROFILE): 'name': 'UPLOAD_FLAGS', 'value': UPLOAD_FLAGS }] - if UPDATE_PLUGINS.lower()=='true': + if USE_PLUGINS.lower()=='true': task_definition["containerDefinitions"][0]["environment"] += [ {"name": "UPDATE_PLUGINS", "value": str(UPDATE_PLUGINS)}, {"name": "PLUGINS_COMMIT", "value": str(PLUGINS_COMMIT)}, {"name": "INSTALL_REQUIREMENTS", "value": str(INSTALL_REQUIREMENTS)}, - {"name": "REQUIREMENTS_FILE", "value": str(REQUIREMENTS_FILE)}, + {"name": "REQUIREMENTS", "value": str(REQUIREMENTS)}, ] return task_definition, taskRoleArn diff --git a/worker/run-worker.sh b/worker/run-worker.sh index 9aef692..be6e733 100644 --- a/worker/run-worker.sh +++ b/worker/run-worker.sh @@ -36,17 +36,35 @@ aws cloudwatch put-metric-alarm --alarm-name ${APP_NAME}_${MY_INSTANCE_ID} --ala python3.8 instance-monitor.py & # 5. UPDATE AND/OR INSTALL PLUGINS -if [[ ${UPDATE_PLUGINS} == 'True' ]]; then +if [[ ${USE_PLUGINS} == 'True' ]]; then + if [[ ${UPDATE_PLUGINS} == 'True' ]]; then + echo "Updating CellProfiler-plugins." cd CellProfiler-plugins git fetch --all - git checkout ${PLUGINS_COMMIT} || echo "No such commit, branch, or version; failing here." & exit 1 cd .. -fi -if [[ ${INSTALL_REQUIREMENTS} == 'True' ]]; then + fi + if [[ ${PLUGINS_COMMIT} != 'False' ]]; then + echo "Checking out specific CellProfiler-plugins commit." cd CellProfiler-plugins - pip install -r ${REQUIREMENTS_FILE} || echo "Requirements file not present or install failed; failing here." & exit 1 + git checkout ${PLUGINS_COMMIT} || echo "No such commit, branch, or version; failing here." & exit 1 cd .. -fi + fi + if [[ ${INSTALL_REQUIREMENTS} == 'True' ]]; then + cd CellProfiler-plugins + if [[ -z "$REQUIREMENTS" ]]; then + REQUIREMENTS = $REQUIREMENTS_FILE + fi + if [[ -d "active_plugins" ]]; then + echo "Installing CellProfiler-plugins requirements." + pip install -e . ${REQUIREMENTS} || echo "Requirements install failed." & exit 1 + cd .. + else + echo "Detected deprecated CellProfiler-plugins repo organization. Installing requirements." + pip install -r ${REQUIREMENTS} || echo "Requirements file not present or install failed; failing here." & exit 1 + cd .. + fi + fi +fi # 6. RUN CP WORKERS for((k=0; k<$DOCKER_CORES; k++)); do From ae170d65e3fb16a1fde9a002897117e5e70a2528 Mon Sep 17 00:00:00 2001 From: ErinWeisbart <54687786+ErinWeisbart@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:54:19 -0700 Subject: [PATCH 3/4] proper truthiness --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 4cbdab5..357f0f1 100644 --- a/run.py +++ b/run.py @@ -17,7 +17,7 @@ CREATE_DASHBOARD = 'False' CLEAN_DASHBOARD = 'False' AUTO_MONITOR = 'False' -REQUIREMENTS_FILE = 'False' +REQUIREMENTS_FILE = False from config import * From 245fe7ba6f2c78d1c82c644921a12078a546c58e Mon Sep 17 00:00:00 2001 From: ErinWeisbart <54687786+ErinWeisbart@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:24:51 -0700 Subject: [PATCH 4/4] make old new, make new old --- documentation/DCP-documentation/step_1_configuration.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/documentation/DCP-documentation/step_1_configuration.md b/documentation/DCP-documentation/step_1_configuration.md index 5100dd2..8ec6e47 100644 --- a/documentation/DCP-documentation/step_1_configuration.md +++ b/documentation/DCP-documentation/step_1_configuration.md @@ -125,6 +125,14 @@ See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_ See [CellProfiler-plugins Documentation](https://plugins.cellprofiler.org/using_plugins.html) for more information on supported flags. For deprecated versions of CellProfiler-plugins before v1.0, pass a path within the CellProfiler-plugins repository to the requirements file you would like to install that will be used in the `pip install -r REQUIREMENTS_FILE` command. +The [CellProfiler/Distributed-CellProfiler Docker](https://hub.docker.com/r/cellprofiler/distributed-cellprofiler/tags) 2.0.0_4.2.4 and older have a clone of the CellProfiler-plugins repository with deprecated organization in them. +If you would like to continue using this clone, set `USE_PLUGINS = 'True'` and `UPDATE_PLUGINS = 'False'`. +Note that if you do need to install requirements with the deprecated organization, pass the path to the requirements file within the CellProfiler-plugins repository as `REQUIREMENTS`. +If you would like to update the CellProfiler-plugins repository with up-to-date plugins and new structure while using the CellProfiler/Distributed-CellProfiler Docker 2.0.0_4.2.4 and older, set `UPDATE_PLUGINS = 'True'`. + +[CellProfiler/Distributed-CellProfiler Dockers](https://hub.docker.com/r/cellprofiler/distributed-cellprofiler/tags) newer than 2.0.0_4.2.4 have current CellProfiler-plugins repository organization. +If you need to use deprecated plugin organization you can access previous commits or version tags by passing them as `PLUGINS_COMMIT`. + *** ### EXAMPLE CONFIGURATIONS