diff --git a/.travis.yml b/.travis.yml index f65940760d..f0e56119c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - TEST=pulp - TEST=docs - TEST=bindings + - TEST=s3 services: - postgresql - redis-server diff --git a/.travis/before_script.sh b/.travis/before_script.sh index 96f59cd948..dc3e496f76 100755 --- a/.travis/before_script.sh +++ b/.travis/before_script.sh @@ -31,23 +31,21 @@ else sed "s/localhost/$(hostname)/g" ../pulpcore/.travis/pulp-smash-config.json > ~/.config/pulp_smash/settings.json fi - -# set up pulp-fixtures docker container -if [[ "$TEST" == 'pulp' ]]; then - docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest -fi -cat ~/.config/pulp_smash/settings.json | \ - jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json -cat temp.json > ~/.config/pulp_smash/settings.json - - -if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' ]]; then +if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' ]]; then # Many tests require pytest/mock, but users do not need them at runtime # (or to add plugins on top of pulpcore or pulp container images.) # So install it here, rather than in the image Dockerfile. $CMD_PREFIX pip3 install pytest mock # Many functional tests require these $CMD_PREFIX dnf install -yq lsof which dnf-plugins-core + + # set up pulp-fixtures docker container + docker run -d -p 0.0.0.0:8000:80 quay.io/pulp/pulp-fixtures:latest + + cat ~/.config/pulp_smash/settings.json | \ + jq "setpath([\"custom\",\"fixtures_origin\"]; \"http://$(hostname):8000/fixtures/\")" > temp.json + cat temp.json > ~/.config/pulp_smash/settings.json + fi if [[ -f $POST_BEFORE_SCRIPT ]]; then diff --git a/.travis/cherrypick.py b/.travis/cherrypick.py index 048ab3bd19..1f15f8bb33 100644 --- a/.travis/cherrypick.py +++ b/.travis/cherrypick.py @@ -91,9 +91,9 @@ def get_merged_commits(pr): if ret.returncode != 0: print(f"Failed to cherry-pick commit {commit.sha}: {ret.stderr.decode('ascii')}") exit(1) - else: - cherrypicks.append(issue) - print(f"Cherry-picked commit {commit.sha}.") + + cherrypicks.append(issue) + print(f"Cherry-picked commit {commit.sha}.") # check if we cherry picked anything if len(cherrypicks) == 0: diff --git a/.travis/install.sh b/.travis/install.sh index 7a0606b88c..783cbe7595 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -69,6 +69,11 @@ images: - $PULP_FILE - $PULP_CERTGUARD VARSYAML + +if [ "$TEST" = 's3' ]; then + echo "s3_test: true" >> vars/vars.yaml +fi + ansible-playbook -v build.yaml cd $TRAVIS_BUILD_DIR/../pulp-operator @@ -93,6 +98,40 @@ spec: admin_password: pulp CRYAML +if [ "$TEST" = 's3' ]; then + cat > deploy/crds/pulpproject_v1alpha1_pulp_cr.yaml << CRYAML + apiVersion: pulpproject.org/v1alpha1 + kind: Pulp + metadata: + name: example-pulp + spec: + pulp_file_storage: + # k3s local-path requires this + access_mode: "ReadWriteOnce" + # We have a little over 40GB free on Travis VMs/instances + size: "40Gi" + image: pulp_file + tag: "${TAG}" + database_connection: + username: pulp + password: pulp + admin_password: pulp + pulp_settings: + aws_access_key_id: "AKIAIT2Z5TDYPX3ARJBA" + aws_secret_access_key: "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" + aws_storage_bucket_name: "pulp3" + aws_default_acl: "@none None" + s3_use_sigv4: true + aws_s3_signature_version: "s3v4" + aws_s3_addressing_style: "path" + aws_s3_region_name: "eu-central-1" + default_file_storage: "storages.backends.s3boto3.S3Boto3Storage" + media_root: '' + aws_s3_endpoint_url: "http://$(hostname):9000" + +CRYAML +fi + # Install k3s, lightweight Kubernetes .travis/k3s-install.sh # Deploy pulp-operator, with the pulp containers, according to CRYAML diff --git a/.travis/pre_before_install.sh b/.travis/pre_before_install.sh index 4852319cfa..ccc1f1aef5 100755 --- a/.travis/pre_before_install.sh +++ b/.travis/pre_before_install.sh @@ -26,3 +26,12 @@ if [ -n "$PULP_CERTGUARD_PR_NUMBER" ]; then fi cd pulpcore + +if [[ "$TEST" == 's3' ]]; then + export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA + export MINIO_SECRET_KEY=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 + docker run -d -p 0.0.0.0:9000:9000 -e MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY -e MINIO_SECRET_KEY=$MINIO_SECRET_KEY minio/minio server /data + wget https://dl.min.io/client/mc/release/linux-amd64/mc + chmod +x mc + sudo mv mc /usr/local/bin +fi diff --git a/.travis/script.sh b/.travis/script.sh index 300d6d3144..5d15d04fe7 100755 --- a/.travis/script.sh +++ b/.travis/script.sh @@ -20,9 +20,9 @@ export FUNC_TEST_SCRIPT=$TRAVIS_BUILD_DIR/.travis/func_test_script.sh export DJANGO_SETTINGS_MODULE=pulpcore.app.settings if [ "$TEST" = 'docs' ]; then - + export PULP_CONTENT_ORIGIN=http://$(hostname):24816 - + cd docs make html @@ -59,7 +59,7 @@ if [ "$TEST" = 'bindings' ]; then rm -rf ./pulpcore-client - ./generate.sh pulpcore ruby + ./generate.sh pulpcore ruby 0 cd pulpcore-client gem build pulpcore_client gem install --both ./pulpcore_client-0.gem @@ -67,7 +67,7 @@ if [ "$TEST" = 'bindings' ]; then rm -rf ./pulp_file-client - ./generate.sh pulp_file ruby + ./generate.sh pulp_file ruby 0 cd pulp_file-client gem build pulp_file_client gem install --both ./pulp_file_client-0.gem @@ -89,6 +89,13 @@ export CMD_STDIN_PREFIX="sudo kubectl exec -i $PULP_API_POD --" cat unittest_requirements.txt | $CMD_STDIN_PREFIX bash -c "cat > /tmp/test_requirements.txt" $CMD_PREFIX pip3 install -r /tmp/test_requirements.txt +if [[ "$TEST" == 's3' ]]; then + mc config host add s3 http://localhost:9000 AKIAIT2Z5TDYPX3ARJBA fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS --api S3v4 + mc config host rm local + mc mb s3/pulp3 --region eu-central-1 + mc tree s3 +fi + # Run unit tests. $CMD_PREFIX bash -c "PULP_DATABASES__default__USER=postgres django-admin test --noinput /usr/local/lib/python${TRAVIS_PYTHON_VERSION}/site-packages/pulpcore/tests/unit/" @@ -107,11 +114,12 @@ export PYTHONPATH=$TRAVIS_BUILD_DIR:${PYTHONPATH} set -u if [[ "$TEST" == "performance" ]]; then + wget -qO- https://github.com/crazy-max/travis-wait-enhanced/releases/download/v1.0.0/travis-wait-enhanced_1.0.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -zxvf - travis-wait-enhanced echo "--- Performance Tests ---" if [[ -z ${PERFORMANCE_TEST+x} ]]; then - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulpcore.tests.performance || show_logs_and_return_non_zero + travis-wait-enhanced --interval=1m --timeout=30m -- pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulpcore.tests.performance || show_logs_and_return_non_zero else - pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulpcore.tests.performance.test_$PERFORMANCE_TEST || show_logs_and_return_non_zero + travis-wait-enhanced --interval=1m --timeout=30m -- pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulpcore.tests.performance.test_$PERFORMANCE_TEST || show_logs_and_return_non_zero fi exit fi diff --git a/containers/images/pulp/Dockerfile.j2 b/containers/images/pulp/Dockerfile.j2 index 36100ebec3..af077a35c3 100644 --- a/containers/images/pulp/Dockerfile.j2 +++ b/containers/images/pulp/Dockerfile.j2 @@ -47,7 +47,11 @@ RUN pip install --upgrade pip RUN mkdir -p /etc/pulp -RUN pip install gunicorn +RUN pip install gunicorn django-storages[boto3] +{% if s3_test is defined %} +# Hacking botocore (https://github.com/boto/botocore/pull/1990): +RUN sed -i "s/hasattr(body, 'read')/getattr(body, '_size', None)/g" $(pip show botocore | grep -i location | awk '{ print $2 }')/botocore/handlers.py +{% endif %} {% if ( item.value.pulpcore is defined and item.value.pulpcore.startswith('./') ) %} diff --git a/pulpcore/tests/unit/models/test_content.py b/pulpcore/tests/unit/models/test_content.py index 8cbd1ccc4d..49da729bcd 100644 --- a/pulpcore/tests/unit/models/test_content.py +++ b/pulpcore/tests/unit/models/test_content.py @@ -1,6 +1,7 @@ import os import tempfile +from django.core.files.storage import default_storage as storage from django.test import TestCase from pulpcore.plugin.models import Artifact, Content, ContentArtifact @@ -23,10 +24,11 @@ def setUp(self): def test_create_and_read_content(self): content = Content.objects.create() content.save() + artifact_file = storage.open(self.artifact01.file.name) content_artifact = ContentArtifact.objects.create( artifact=self.artifact01, content=content, - relative_path=self.artifact01.file.path) + relative_path=artifact_file.name) content_artifact.save() self.assertTrue( Content.objects.filter(pk=content.pk).exists() diff --git a/template_config.yml b/template_config.yml index e1208f055b..3bcc12bb44 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,14 +1,11 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# pulp_file is hardcoded in plugin_template when generating for pulpcore in some -# places. In other places, plugin_template uses generic logic for it, so we -# specify it here. -# Also, .travis/pre_before_install.sh reads the pulp_file branch. At runtime, -# not template generation time. additional_plugins: -- { name: pulp_file, branch: master } -- { name: pulp-certguard, branch: master } +- branch: master + name: pulp_file +- branch: master + name: pulp-certguard black: false check_commit_message: true cherry_pick_automation: true @@ -31,11 +28,14 @@ plugin_name: pulpcore plugin_snake: pulpcore pulp_settings: null pulpcore_branch: null +pulpcore_pip_version_specifier: null pydocstyle: true pypi_username: pulp -stable_branch: "3.2" +stable_branch: '3.2' test_bindings: true test_performance: false +test_s3: true +travis_addtl_services: [] travis_notifications: irc: channels: