Skip to content

Commit

Permalink
Merging with dev
Browse files Browse the repository at this point in the history
  • Loading branch information
anuprulez committed Dec 11, 2016
2 parents ba56bd5 + 06de9d0 commit 887fe9c
Show file tree
Hide file tree
Showing 626 changed files with 7,487 additions and 6,563 deletions.
53 changes: 28 additions & 25 deletions .ci/flake8_lint_include_list.txt
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ lib/galaxy/sample_tracking/__init__.py
lib/galaxy/sample_tracking/sample.py
lib/galaxy/security/validate_user_input.py
lib/galaxy/tags/
lib/galaxy/tools/actions/metadata.py
lib/galaxy/tools/actions/
lib/galaxy/tools/cwl/
lib/galaxy/tools/data_manager/__init__.py
lib/galaxy/tools/deps/
Expand Down Expand Up @@ -319,6 +319,7 @@ lib/galaxy/web/base/controllers/__init__.py
lib/galaxy/web/base/__init__.py
lib/galaxy/web/buildapp.py
lib/galaxy/web/formatting.py
lib/galaxy/web/framework/base.py
lib/galaxy/web/framework/decorators.py
lib/galaxy/web/framework/helpers/grids.py
lib/galaxy/web/framework/__init__.py
Expand Down Expand Up @@ -356,6 +357,7 @@ lib/galaxy/webapps/galaxy/controllers/tool_runner.py
lib/galaxy/webapps/galaxy/controllers/userskeys.py
lib/galaxy/webapps/galaxy/__init__.py
lib/galaxy/webapps/__init__.py
lib/galaxy/webapps/reports/config.py
lib/galaxy/webapps/reports/controllers/__init__.py
lib/galaxy/webapps/reports/controllers/query.py
lib/galaxy/webapps/reports/__init__.py
Expand Down Expand Up @@ -422,11 +424,7 @@ scripts/build_toolbox.py
scripts/build_universe_config.py
scripts/check_galaxy.py
scripts/check_python.py
scripts/cleanup_datasets/cleanup_datasets.py
scripts/cleanup_datasets/populate_uuid.py
scripts/cleanup_datasets/remove_renamed_datasets_from_disk.py
scripts/cleanup_datasets/rename_purged_datasets.py
scripts/cleanup_datasets/update_metadata.py
scripts/cleanup_datasets/
scripts/create_db.py
scripts/data_libraries/
scripts/db_shell.py
Expand Down Expand Up @@ -482,7 +480,6 @@ test/api/test_libraries.py
test/api/test_page_revisions.py
test/api/test_search.py
test/api/test_tool_data.py
test/api/test_tools.py
test/api/test_tours.py
test/api/test_workflow_extraction.py
test/api/test_workflows_from_yaml.py
Expand All @@ -502,63 +499,69 @@ test/casperjs/
test/functional/database_contexts.py
test/functional/__init__.py
test/functional/test_data_managers.py
test/functional/test_toolbox.py
test/functional/tool-data/
test/functional/tools/
test/functional/workflow.py
test/integration/
test/manual/__init__.py
test/shed_functional/base/common.py
test/shed_functional/base/__init__.py
test/shed_functional/functional/
test/shed_functional/functional_tests.py
test/shed_functional/__init__.py
test/unit/dataset_collections/__init__.py
test/unit/dataset_collections/
test/unit/datatypes/dataproviders/__init__.py
test/unit/datatypes/dataproviders/test_line_dataproviders.py
test/unit/datatypes/__init__.py
test/unit/datatypes/test_data.py
test/unit/jobs/dynamic_tool_destination/__init__.py
test/unit/jobs/dynamic_tool_destination/mockGalaxy.py
test/unit/jobs/dynamic_tool_destination/ymltests.py
test/unit/jobs/dynamic_tool_destination/
test/unit/jobs/__init__.py
test/unit/jobs/test_command_factory.py
test/unit/jobs/test_datasets.py
test/unit/jobs/test_job_configuration.py
test/unit/jobs/test_job_output_checker.py
test/unit/jobs/test_mapper.py
test/unit/jobs/test_rule_helper.py
test/unit/jobs/test_rules/
test/unit/jobs/test_runner_params.py
test/unit/managers/__init__.py
test/unit/managers/test_CollectionManager.py
test/unit/managers/test_DatasetManager.py
test/unit/managers/test_HDAManager.py
test/unit/managers/test_HDCAManager.py
test/unit/managers/test_HistoryContentsManager.py
test/unit/managers/test_HistoryManager.py
test/unit/managers/test_UserManager.py
test/unit/shed_unit/__init__.py
test/unit/shed_unit/test_fabric_util.py
test/unit/shed_unit/test_td_common_util.py
test/unit/test_galaxy_transactions.py
test/unit/test_lazy_process.py
test/unit/test_routes.py
test/unit/test_security_helper.py
test/unit/test_sockets.py
test/unit/test_sqlite_utils.py
test/unit/test_topsort.py
test/unit/tools/filter_modules/
test/unit/tools/__init__.py
test/unit/tools/test_data_parameters.py
test/unit/tools/test_tool_dependency_description.py
test/unit/tools/test_tool_panel.py
test/unit/tools/test_citations.py
test/unit/tools/test_conda_resolution.py
test/unit/tools/test_parsing.py
test/unit/tools/test_select_parameters.py
test/unit/tools/test_toolbox_filters.py
test/unit/tools/test_tool_dependency_description.py
test/unit/tools/test_tool_deps.py
test/unit/tools/test_tool_external_files.py
test/unit/tools/test_tool_loader.py
test/unit/tools/test_toolbox_filters.py
test/unit/tools/test_tool_panel.py
test/unit/tools/test_watcher.py
test/unit/tools/test_wrappers.py
test/unit/tools_support.py
test/unit/unittest_utils/__init__.py
test/unit/unittest_utils/utility.py
test/unit/unittest_utils/
test/unit/visualizations/__init__.py
test/unit/visualizations/plugins/__init__.py
test/unit/web/base/__init__.py
test/unit/web/framework/__init__.py
test/unit/web/__init__.py
test/unit/workflows/__init__.py
test/unit/workflows/test_extract_summary.py
test/unit/workflows/test_modules.py
test/unit/workflows/test_render.py
test/unit/workflows/test_workflow_progress.py
test/unit/workflows/
test/unit/test_objectstore.py
tool_list.py
tools/
3 changes: 3 additions & 0 deletions .ci/jenkins/api/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc -api "$@"
3 changes: 3 additions & 0 deletions .ci/jenkins/framework/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --framework "$@"
5 changes: 5 additions & 0 deletions .ci/jenkins/integration/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash

./scripts/common_startup.sh --dev-wheels

nosetests test/integration
24 changes: 24 additions & 0 deletions .ci/jenkins/selenium/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
FROM toolshed/requirements
MAINTAINER John Chilton, jmchilton@gmail.com

RUN apt-get -qq update && \
apt-get install --no-install-recommends -y postgresql-client python-pip && \
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

ENV GALAXY_ROOT=/galaxy \
GALAXY_CONFIG_OVERRIDE_ADMIN_USERS=admin@galaxy.org \
GALAXY_CONFIG_OVERRIDE_ALLOW_USER_DATASET_PURGE=true \
GALAXY_CONFIG_OVERRIDE_ALLOW_LIBRARY_PATH_PASTE=true \
GALAXY_CONFIG_OVERRIDE_ENABLE_BETA_WORKFLOW_MODULES=true \
GALAXY_CONFIG_OVERRIDE_ALLOW_USER_DELETION=true \
GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=postgresql://postgres@postgres/galaxy \
GALAXY_CONFIG_OVERRIDE_MASTER_API_KEY=94a548bea347a35e457a804bf75bec53 \
GALAXY_CONFIG_OVERRIDE_JOB_CONFIG_FILE=/galaxy/config/job_conf.xml.sample_basic \
GALAXY_CONFIG_OVERRIDE_DATATYPES_CONFIG_FILE=/galaxy/test/functional/tools/sample_datatypes_conf.xml \
GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE=/galaxy/test/functional/tools/samples_tool_conf.xml \
GALAXY_CONFIG_FILE=/galaxy/.ci/jenkins/selenium/galaxy.ini \
GALAXY_VIRTUAL_ENV=/galaxy_env

ADD run_galaxy.bash /run_galaxy.bash
EXPOSE 8080
CMD /run_galaxy.bash
19 changes: 19 additions & 0 deletions .ci/jenkins/selenium/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
version: '2'
services:
postgres:
image: postgres
galaxy:
build: .
image: ubuntu
volumes:
- "${TARGET_ROOT}:${TARGET_PATH}"
links:
- postgres
ports:
- "${GALAXY_PORT}:8080"
selenium:
image: selenium/standalone-chrome:3.0.1-aluminum
ports:
- "${SELENIUM_PORT}:4444"
links:
- galaxy
11 changes: 11 additions & 0 deletions .ci/jenkins/selenium/galaxy.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[server:main]

use = egg:Paste#http
port = 8080
host = 0.0.0.0
use_threadpool = True
threadpool_kill_thread_limit = 10800

[app:main]

paste.app_factory = galaxy.web.buildapp:app_factory
16 changes: 16 additions & 0 deletions .ci/jenkins/selenium/run_galaxy.bash
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/bash

set -e

sleep 30 # TODO: wait on something instead of just sleeping...

echo `df`

echo "Creating postgres database for Galaxy"
createdb -w -U postgres -h postgres galaxy

echo "Starting and waiting for Galaxy daemon(s)"
GALAXY_RUN_ALL=1 bash "$GALAXY_ROOT/run.sh" --daemon --wait

echo "Galaxy daemon ready, monitoring Galaxy logs"
tail -f "$GALAXY_ROOT/main.log"
110 changes: 110 additions & 0 deletions .ci/jenkins/selenium/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
#!/bin/bash

TEST_DIRECTORY=`dirname $0`

DEFAULT_COMPOSE_PROJECT_NAME=`basename $TEST_DIRECTORY`
COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-$DEFAULT_COMPOSE_PROJECT_NAME}
# If in Jenkins environment, append ${BUILD_NUMBER} to project so builds don't interfer.
if [ ! -z "$BUILD_NUMBER" ];
then
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME}${BUILD_NUMBER}"
fi
export COMPOSE_PROJECT_NAME

echo "Running Jenkins test from $TEST_DIRECTORY with compose project name $COMPOSE_PROJECT_NAME"

find lib -iname '*pyc' -exec rm -rf {} \;
find test -iname '*pyc' -exec rm -rf {} \;

./scripts/common_startup.sh --dev-wheels

. .venv/bin/activate

pip install docker-compose

# TODO: Let docker-compose pick these at random.
export GALAXY_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'`
export SELENIUM_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'`

export TARGET_ROOT=`pwd`
export TARGET_PATH=/galaxy

cd $TEST_DIRECTORY

echo "Cleaning up previous executions if needed."
docker-compose down | true
docker-compose build galaxy
docker-compose up -d

for service_name in postgres galaxy selenium
do
echo "Waiting on service ${service_name}"
while true
do
if docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}"
then
container_id=`docker ps | grep "${COMPOSE_PROJECT_NAME}_${service_name}" | cut -d " " -f 1`
eval "${service_name}_container_id=${container_id}"
echo "Service ${service_name} ready - with container ID ${container_id}"
break
fi
printf "."
sleep 1;
done
done


if [ "$1" = "--debug-running-containers" ];
then
env
exit 0
fi

export GALAXY_TEST_EXTERNAL="http://localhost:$GALAXY_PORT/"

echo "Waiting on docker-compose managed Galaxy server - $GALAXY_TEST_EXTERNAL."
while ! curl -s "$GALAXY_TEST_EXTERNAL";
do
for service_name in postgres galaxy selenium
do
if ! docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}"
then
echo "Service ${service_name} stopped before Galaxy came up, exiting and halting containers."
for service_name in postgres galaxy selenium
do
container_id_var="${service_name}_container_id"
container_id="${!container_id_var}"
echo "Dumping logs for $service_name container (${container_id})..."
echo "---"
docker logs "${container_id}"
echo "---"
done
docker-compose down | true
exit 1
fi
done

printf "."
sleep 4;
done;

# Access Selenium on localhost via port $SELENIUM_PORT
export GALAXY_TEST_SELENIUM_REMOTE=1
export GALAXY_TEST_SELENIUM_REMOTE_PORT="${SELENIUM_PORT}"

# Access Galaxy on localhost via port $GALAXY_PORT
export GALAXY_TEST_PORT="${GALAXY_PORT}"

# Have Selenium access Galaxy at this URL
export GALAXY_TEST_EXTERNAL_FROM_SELENIUM="http://galaxy:8080"

cd ../../..

./run_tests.sh --selenium "$@"
exit_code=$?

cd $TEST_DIRECTORY

docker-compose down

exit $exit_code
3 changes: 3 additions & 0 deletions .ci/jenkins/toolshed/run_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/bin/bash

./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --toolshed "$@"

0 comments on commit 887fe9c

Please sign in to comment.