From ba45c4ef900da1232365df240dedbd8a0c852f1d Mon Sep 17 00:00:00 2001 From: Fabricio Aguiar Date: Mon, 2 Mar 2020 18:11:32 -0300 Subject: [PATCH] Running tests on minio Required PR: https://github.com/pulp/pulp-operator/pull/35 Required PR: https://github.com/pulp/pulpcore/pull/568 https://pulp.plan.io/issues/6155 ref #6155 --- .travis.yml | 1 + .travis/before_install.sh | 13 +++++++++-- .travis/before_script.sh | 20 ++++++++--------- .travis/cherrypick.py | 2 +- .travis/install.sh | 40 ++++++++++++++++++++++++++++++++++ .travis/publish_client_gem.sh | 5 ++++- .travis/publish_client_pypi.sh | 5 ++++- .travis/script.sh | 7 ++++++ CHANGES/6155.misc | 1 + template_config.yml | 22 ++++++++++++++++--- 10 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 CHANGES/6155.misc diff --git a/.travis.yml b/.travis.yml index 38a93499cc..b0c2166cc2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - TEST=pulp - TEST=docs - TEST=bindings + - TEST=s3 services: - postgresql - redis-server diff --git a/.travis/before_install.sh b/.travis/before_install.sh index 81989d1f3f..afce80fb26 100755 --- a/.travis/before_install.sh +++ b/.travis/before_install.sh @@ -84,12 +84,21 @@ fi git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master +cd pulpcore if [ -n "$PULP_PR_NUMBER" ]; then - cd pulpcore git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER git checkout $PULP_PR_NUMBER - cd .. fi +if [[ "$TEST" == 's3' ]]; then + export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + export MINIO_SECRET_KEY=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 + docker run -d -p 0.0.0.0:9000:9000 -e MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -e MINIO_SECRET_KEY=$MINIO_SECRET_KEY minio/minio server /data + wget https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + sudo mv mc /usr/local/bin +fi + +cd .. diff --git a/.travis/before_script.sh b/.travis/before_script.sh index 96f59cd948..dc3e496f76 100755 --- a/.travis/before_script.sh +++ b/.travis/before_script.sh @@ -31,23 +31,21 @@ else sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json fi - -# set up pulp-fixtures docker container -if [[ "$TEST" == 'pulp' ]]; then - docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest -fi -cat ~/.config/pulp_smash/settings.json | \ - jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json -cat temp.json > ~/.config/pulp_smash/settings.json - - -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then # Many tests require pytest/mock, but users do not need them at runtime # (or to add plugins on top of pulpcore or pulp container images.) # So install it here, rather than in the image Dockerfile. $CMD_PREFIX pip3 install pytest mock # Many functional tests require these $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core + + # set up pulp-fixtures docker container + docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest + + cat ~/.config/pulp_smash/settings.json | \ + jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json + cat temp.json > ~/.config/pulp_smash/settings.json + fi if [[ -f $POST_BEFORE_SCRIPT ]]; then diff --git a/.travis/cherrypick.py b/.travis/cherrypick.py index f3da465a0c..048ab3bd19 100644 --- a/.travis/cherrypick.py +++ b/.travis/cherrypick.py @@ -67,7 +67,7 @@ def get_merged_commits(pr): g = Github(GITHUB_TOKEN) grepo = g.get_repo(REPOSITORY) (label,) = (l for l in grepo.get_labels() if l.name == PR_LABEL) -issues = grepo.get_issues(labels=[label], state="all") +issues = grepo.get_issues(labels=[label], state="all", sort="updated", direction="asc") cherrypicks = [] diff --git a/.travis/install.sh b/.travis/install.sh index 8f931c8710..2d648273bd 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -54,6 +54,7 @@ images: - pulp_rpm-${TAG}: image_name: pulp_rpm tag: $TAG + pulpcore: pulpcore plugins: - ./pulp_rpm VARSYAML @@ -69,6 +70,11 @@ images: - ./pulp_rpm VARSYAML fi + +if [ "$TEST" = 's3' ]; then + echo "s3_test: true" >> vars/vars.yaml +fi + ansible-playbook -v build.yaml cd $TRAVIS_BUILD_DIR/../pulp-operator @@ -93,6 +99,40 @@ spec: admin_password: pulp CRYAML +if [ "$TEST" = 's3' ]; then + cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML + apiVersion: pulpproject.org/v1alpha1 + kind: Pulp + metadata: + name: example-pulp + spec: + pulp_file_storage: + # k3s local-path requires this + access_mode: "ReadWriteOnce" + # We have a little over 40GB free on Travis VMs/instances + size: "40Gi" + image: pulp_rpm + tag: "${TAG}" + database_connection: + username: pulp + password: pulp + admin_password: pulp + pulp_settings: + aws_access_key_id: "AKIAIT2Z5TDYPX3ARJBA" + aws_secret_access_key: "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" + aws_storage_bucket_name: "pulp3" + aws_default_acl: "@none None" + s3_use_sigv4: true + aws_s3_signature_version: "s3v4" + aws_s3_addressing_style: "path" + aws_s3_region_name: "eu-central-1" + default_file_storage: "storages.backends.s3boto3.S3Boto3Storage" + media_root: '' + aws_s3_endpoint_url: "http://$(hostname):9000" + +CRYAML +fi + # Install k3s, lightweight Kubernetes .travis/k3s-install.sh # Deploy pulp-operator, with the pulp containers, according to CRYAML diff --git a/.travis/publish_client_gem.sh b/.travis/publish_client_gem.sh index 718996ff7f..396c87debc 100755 --- a/.travis/publish_client_gem.sh +++ b/.travis/publish_client_gem.sh @@ -23,7 +23,10 @@ if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} else # Daily publishing of development version (ends in ".dev" reported as ".dev0") - [ "${REPORTED_VERSION%.dev*}" != "${REPORTED_VERSION}" ] || exit 1 + if [ "${REPORTED_VERSION%.dev*}" == "${REPORTED_VERSION}" ]; then + echo "Refusing to publish bindings. $REPORTED_VERSION does not contain 'dev'." + exit 1 + fi export EPOCH="$(date +%s)" export VERSION=${REPORTED_VERSION}${EPOCH} fi diff --git a/.travis/publish_client_pypi.sh b/.travis/publish_client_pypi.sh index 7f616445ee..35b90eabbb 100755 --- a/.travis/publish_client_pypi.sh +++ b/.travis/publish_client_pypi.sh @@ -21,7 +21,10 @@ if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then export VERSION=${REPORTED_VERSION} else # Daily publishing of development version (ends in ".dev" reported as ".dev0") - [ "${REPORTED_VERSION%.dev*}" != "${REPORTED_VERSION}" ] || exit 1 + if [ "${REPORTED_VERSION%.dev*}" == "${REPORTED_VERSION}" ]; then + echo "Refusing to publish bindings. $REPORTED_VERSION does not contain 'dev'." + exit 1 + fi export EPOCH="$(date +%s)" export VERSION=${REPORTED_VERSION}${EPOCH} fi diff --git a/.travis/script.sh b/.travis/script.sh index 5533a7676e..fd4a67807d 100755 --- a/.travis/script.sh +++ b/.travis/script.sh @@ -89,6 +89,13 @@ export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --" cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt" $CMD_PREFIX pip3 install -r /tmp/test_requirements.txt +if [[ "$TEST" == 's3' ]]; then + mc config host add s3 http://localhost:9000 AKIAIT2Z5TDYPX3ARJBA fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS --api S3v4 + mc config host rm local + mc mb s3/pulp3 --region eu-central-1 + mc tree s3 +fi + # Run unit tests. $CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulp_rpm/tests/unit/" diff --git a/CHANGES/6155.misc b/CHANGES/6155.misc new file mode 100644 index 0000000000..aa4e6c15ba --- /dev/null +++ b/CHANGES/6155.misc @@ -0,0 +1 @@ +Running tests on minio diff --git a/template_config.yml b/template_config.yml index 4877c35b08..e3e0b6d7e5 100644 --- a/template_config.yml +++ b/template_config.yml @@ -24,12 +24,28 @@ plugin_name: pulp_rpm plugin_snake: pulp_rpm pulp_settings: null pulpcore_branch: master +pulpcore_pip_version_specifier: null pydocstyle: true pypi_username: pulp +s3_settings: + aws_access_key_id: '"AKIAIT2Z5TDYPX3ARJBA"' + aws_default_acl: '"@none None"' + aws_s3_addressing_style: '"path"' + aws_s3_endpoint_url: '"http://$(hostname):9000"' + aws_s3_region_name: '"eu-central-1"' + aws_s3_signature_version: '"s3v4"' + aws_secret_access_key: '"fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS"' + aws_storage_bucket_name: '"pulp3"' + default_file_storage: '"storages.backends.s3boto3.S3Boto3Storage"' + media_root: '' + s3_use_sigv4: 'true' stable_branch: '3.0' test_bindings: true test_performance: - - sync - - publish - - pulp_to_pulp +- sync +- publish +- pulp_to_pulp +test_s3: true +travis_addtl_services: [] travis_notifications: None +