-
Notifications
You must be signed in to change notification settings - Fork 967
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Workflow scaling test infrastructure.
- Add script to hit the API with workflows requests. - Update testing Dockerfile to allow just booting up with a fixed master API key for this test to leverage. - WIP: on updating test Dockerfile to work with slurm, multiple handlers, uwsgi, etc...
- Loading branch information
Showing
10 changed files
with
358 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| from __future__ import print_function | ||
|
|
||
| import os | ||
| import sys | ||
|
|
||
| script_dir = os.path.dirname(__file__) | ||
| galaxy_root = os.path.join(script_dir, os.path.pardir) | ||
| new_path = [ os.path.join( galaxy_root, "lib" ) ] | ||
| new_path.extend( sys.path[1:] ) | ||
| sys.path = new_path | ||
|
|
||
| try: | ||
| from argparse import ArgumentParser | ||
| except ImportError: | ||
| ArgumentParser = None | ||
| import re | ||
|
|
||
| try: | ||
| from galaxy import eggs | ||
| eggs.require("numpy") | ||
| except ImportError: | ||
| pass | ||
| import numpy | ||
|
|
||
|
|
||
| DESCRIPTION = "" | ||
|
|
||
| TIMING_LINE_PATTERN = re.compile("\((\d+.\d+) ms\)") | ||
|
|
||
|
|
||
| def main(argv=None): | ||
| if ArgumentParser is None: | ||
| raise Exception("Test requires Python 2.7") | ||
| arg_parser = ArgumentParser(description=DESCRIPTION) | ||
| arg_parser.add_argument("--file", default="paster.log") | ||
| arg_parser.add_argument("--print_lines", default=False, action="store_true") | ||
| arg_parser.add_argument("--pattern") | ||
|
|
||
| args = arg_parser.parse_args(argv) | ||
| print_lines = args.print_lines | ||
| filter_pattern = re.compile(args.pattern) | ||
| times = [] | ||
| for line in open(args.file, "r"): | ||
| if not filter_pattern.search(line): | ||
| continue | ||
|
|
||
| match = TIMING_LINE_PATTERN.search(line) | ||
| if not match: | ||
| continue | ||
|
|
||
| times.append(float(match.group(1))) | ||
| if print_lines: | ||
| print(line.strip()) | ||
|
|
||
| template = "Summary (ms) - Mean: %f, Median: %f, Max: %f, Min: %f, StdDev: %f" | ||
| message = template % ( | ||
| numpy.mean(times), | ||
| numpy.median(times), | ||
| numpy.max(times), | ||
| numpy.min(times), | ||
| numpy.std(times) | ||
| ) | ||
| print(message) | ||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,23 +1,58 @@ | ||
| #!/bin/bash | ||
| set -e | ||
|
|
||
| # TODO: switch on environment to boot mysql here... | ||
| GALAXY_TEST_DATABASE_TYPE=${GALAXY_TEST_DATABASE_TYPE:-"postgres"} | ||
| if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; | ||
| then | ||
| su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres | ||
| sleep 3 | ||
| export GALAXY_TEST_DBURI="postgres://galaxy@localhost:5930/galaxy" | ||
| sleep 3 | ||
| GALAXY_TEST_DBURI="postgres://root@localhost:5930/galaxy" | ||
| elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ]; | ||
| then | ||
| sh /opt/galaxy/start_mysql.sh | ||
| export GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" | ||
| GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" | ||
| elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ]; | ||
| then | ||
| export GALAXY_TEST_DBURI="sqlite:////opt/galaxy/galaxy.sqlite" | ||
| GALAXY_TEST_DBURI="sqlite:////opt/galaxy/galaxy.sqlite" | ||
| else | ||
| echo "Unknown database type" | ||
| exit 1 | ||
| fi | ||
| export GALAXY_TEST_DBURI | ||
|
|
||
| cd /galaxy | ||
| GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" sh manage_db.sh upgrade | ||
| sh run_tests.sh $@ | ||
| GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI"; | ||
| export GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION | ||
|
|
||
| sh manage_db.sh upgrade | ||
|
|
||
| if [ -z "$GALAXY_NO_TESTS" ]; | ||
| then | ||
| sh run_tests.sh $@ | ||
| else | ||
| GALAXY_CONFIG_MASTER_API_KEY=${GALAXY_CONFIG_MASTER_API_KEY:-"testmasterapikey"} | ||
| GALAXY_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/galaxy.ini.sample} | ||
| GALAXY_CONFIG_CHECK_MIGRATE_TOOLS=false | ||
| if [ -z "$GALAXY_MULTI_PROCESS" ]; | ||
| then | ||
| GALAXY_CONFIG_JOB_CONFIG_FILE=${GALAXY_CONFIG_JOB_CONFIG_FILE:-config/job_conf.xml.sample} | ||
| else | ||
| GALAXY_CONFIG_JOB_CONFIG_FILE=/etc/galaxy/job_conf.xml | ||
| fi | ||
| GALAXY_CONFIG_FILE_PATH=${GALAXY_CONFIG_FILE_PATH:-/tmp/gx1} | ||
| GALAXY_CONFIG_NEW_FILE_PATH=${GALAXY_CONFIG_NEW_FILE_PATH:-/tmp/gxtmp} | ||
|
|
||
| export GALAXY_CONFIG_MASTER_API_KEY | ||
| export GALAXY_CONFIG_FILE | ||
| export GALAXY_CONFIG_CHECK_MIGRATE_TOOLS | ||
| export GALAXY_CONFIG_JOB_CONFIG_FILE | ||
| export GALAXY_CONFIG_FILE_PATH | ||
| export GALAXY_CONFIG_NEW_FILE_PATH | ||
|
|
||
| if [ -z "$GALAXY_MULTI_PROCESS" ]; | ||
| then | ||
| sh run.sh $@ | ||
| else | ||
| /usr/bin/supervisord | ||
| fi | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| <tool id="cat_interleave" name="Interleave two inputs (for test workflows)"> | ||
| <command> | ||
| cat $input1 $input2 > $out_file1; | ||
| cat $input2 $input1 > $out_file2; | ||
| </command> | ||
| <inputs> | ||
| <param name="input1" type="data" label="Input 1"/> | ||
| <param name="input2" type="data" label="Input 2"/> | ||
| </inputs> | ||
| <outputs> | ||
| <data name="out_file1" format="input" metadata_source="input1"/> | ||
| <data name="out_file2" format="input" metadata_source="input2"/> | ||
| </outputs> | ||
| <tests> | ||
| </tests> | ||
| <help> | ||
| </help> | ||
| </tool> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
Oops, something went wrong.