Upgrade tests

https://pulp.plan.io/issues/8719
closes #8719

.github/workflows/ci.yml

@@ -128,3 +128,77 @@ jobs:
           docker exec pulp ls -latr /etc/yum.repos.d/ || true
           docker exec pulp cat /etc/yum.repos.d/* || true
           docker exec pulp pip3 list
+
+  upgrade:
+    runs-on: ubuntu-latest
+    # run only after lint finishes
+    needs: lint
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - FROM_BRANCH: 1.4
+            FROM_PULPCORE_BRANCH: master
+          - FROM_BRANCH: 1.5
+            FROM_PULPCORE_BRANCH: master
+          - FROM_BRANCH: 1.6
+            FROM_PULPCORE_BRANCH: master
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # by default, it uses a depth of 1
+          # this fetches all history so that we can read each commit
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
+
+      - name: Install httpie
+        run: |
+          echo ::group::HTTPIE
+          sudo apt-get update -yq
+          sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie
+          echo ::endgroup::
+          echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
+          echo "TEST=upgrade" >> $GITHUB_ENV
+          echo "FROM_BRANCH=${{ matrix.FROM_BRANCH }}" >> $GITHUB_ENV
+          echo "FROM_PULPCORE_BRANCH=${{ matrix.FROM_PULPCORE_BRANCH }}" >> $GITHUB_ENV
+
+      - name: Before Install
+        run: .github/workflows/scripts/before_install.sh
+        shell: bash
+
+      - name: Install
+        run: .github/workflows/scripts/install.sh
+        env:
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+        shell: bash
+
+      - name: Before Script
+        run: |
+          .github/workflows/scripts/before_script.sh
+
+      - name: Setting secrets
+        if: github.event_name != 'pull_request'
+        run: python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT"
+        env:
+          SECRETS_CONTEXT: ${{ toJson(secrets) }}
+
+      - name: Script
+        run: .github/workflows/scripts/script.sh
+        shell: bash
+
+      - name: After failure
+        if: failure()
+        run: |
+          echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
+          http --timeout 30 --check-status --pretty format --print hb http://pulp/pulp/api/v3/status/ || true
+          docker images || true
+          docker ps -a || true
+          docker logs pulp || true
+          docker exec pulp ls -latr /etc/yum.repos.d/ || true
+          docker exec pulp cat /etc/yum.repos.d/* || true
+          docker exec pulp pip3 list

.github/workflows/scripts/before_install.sh

@@ -70,6 +70,9 @@ else
   export CI_BASE_IMAGE=
 fi
 
+FROM_BRANCH=${FROM_BRANCH:-master}
+git checkout $FROM_BRANCH
+
 cd ..
 
 git clone --depth=1 https://github.com/pulp/pulp-openapi-generator.git
@@ -99,8 +102,8 @@ pip install -e .
 pulp config create --base-url http://pulp --location tests/settings.toml
 cd ..
 
-
-git clone --depth=1 https://github.com/pulp/pulpcore.git --branch master
+FROM_PULPCORE_BRANCH=${FROM_PULPCORE_BRANCH:-master}
+git clone --depth=1 https://github.com/pulp/pulpcore.git --branch $FROM_PULPCORE_BRANCH
 
 cd pulpcore
 if [ -n "$PULPCORE_PR_NUMBER" ]; then

.github/workflows/scripts/before_script.sh

@@ -29,7 +29,7 @@ tail -v -n +1 .ci/ansible/vars/main.yaml
 echo "PULP CONFIG:"
 tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
 
-if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then
+if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "$TEST" == 's3' || "$TEST" == "plugin-from-pypi" ]]; then
   # Many functional tests require these
   cmd_prefix dnf install -yq lsof which dnf-plugins-core
 fi

CHANGES/8719.misc

@@ -0,0 +1 @@
+Inroduce upgrade tests

template_config.yml

@@ -46,6 +46,13 @@ pulp_settings:
   - /tmp
 pulpcore_branch: master
 pulpcore_pip_version_specifier: null
+upgrade_range:
+  - plugin_branch: 1.6
+    pulpcore_branch: master
+  - plugin_branch: 1.5
+    pulpcore_branch: master
+  - plugin_branch: 1.4
+    pulpcore_branch: master
 pulpprojectdotorg_key_id: aa499d7938ed
 pydocstyle: true
 pypi_username: pulp