CI update

.ci/ansible/smash-config.json

@@ -6,7 +6,7 @@
     ],
     "selinux enabled": false,
     "version": "3",
-    "aiohttp_fixtures_origin": "172.18.0.1"
+    "aiohttp_fixtures_origin": "127.0.0.1"
   },
   "hosts": [
     {
@@ -26,7 +26,7 @@
         "pulp workers": {},
         "redis": {},
         "shell": {
-          "transport": "docker"
+          "transport": "local"
         }
       }
     }

.ci/ansible/start_container.yaml

@@ -41,7 +41,7 @@
         image: "{{ item.image }}"
         auto_remove: true
         recreate: true
-        privileged: false
+        privileged: true
         networks:
           - name: pulp_ci_bridge
             aliases: "{{ item.name }}"

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-139-g0d40f35
+2021.08.26-146-gae643c8

.github/workflows/ci.yml

@@ -133,6 +133,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
 
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+
       - name: Install
 
         run: .github/workflows/scripts/install.sh
@@ -152,11 +155,6 @@ jobs:
         run: .github/workflows/scripts/install_python_client.sh
         shell: bash
 
-      - name: Install Ruby client
-        if: ${{ env.TEST == 'bindings' }}
-        run: .github/workflows/scripts/install_ruby_client.sh
-        shell: bash
-
       - name: Before Script
 
         run: .github/workflows/scripts/before_script.sh

.github/workflows/release.yml

@@ -341,6 +341,7 @@ jobs:
           set -euv
           export COMMIT_MSG=$(git log --format=%B --no-merges -1)
           export GH_ISSUES=$(echo $COMMIT_MSG | grep -o "GH Issues: .*" | awk '{print $3}')
+          pip install pygithub
 
           echo "GH Issues $GH_ISSUES"
           python .ci/scripts/update_github.py

.github/workflows/scripts/install.sh

@@ -22,9 +22,6 @@ if [[ "$TEST" = "docs" || "$TEST" = "publish" ]]; then
   pip install -r doc_requirements.txt
 fi
 
-pip install -e ../pulpcore -e ../pulp_file -e ../pulp-certguard
-pip install -r functest_requirements.txt
-
 cd .ci/ansible/
 
 TAG=ci_build
@@ -62,6 +59,8 @@ plugins:
     source: pulp_file
   - name: pulp-certguard
     source: pulp-certguard
+  - name: pulp-smash
+    source: ./pulp-smash
 VARSYAML
 else
   cat >> vars/main.yaml << VARSYAML
@@ -77,6 +76,8 @@ plugins:
     source: $PULP_CERTGUARD
   - name: pulpcore
     source: "${PULPCORE}"
+  - name: pulp-smash
+    source: ./pulp-smash
 VARSYAML
 fi
 
@@ -87,6 +88,8 @@ services:
     volumes:
       - ./settings:/etc/pulp
       - ./ssh:/keys/
+      - ~/.config:/root/.config
+      - ../../../pulp-openapi-generator:/root/pulp-openapi-generator
 VARSYAML
 
 cat >> vars/main.yaml << VARSYAML

.github/workflows/scripts/install_python_client.sh

@@ -7,7 +7,7 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
-set -euv
+set -mveuo pipefail
 
 export PULP_URL="${PULP_URL:-http://pulp}"
 
@@ -30,7 +30,7 @@ export response=$(curl --write-out %{http_code} --silent --output /dev/null http
 if [ "$response" == "200" ];
 then
   echo "pulp_rpm client $VERSION has already been released. Installing from PyPI."
-  pip install pulp-rpm-client==$VERSION
+  docker exec pulp pip3 install pulp-rpm-client==$VERSION
   mkdir -p dist
   tar cvf python-client.tar ./dist
   exit
@@ -41,7 +41,7 @@ rm -rf pulp_rpm-client
 ./generate.sh pulp_rpm python $VERSION
 cd pulp_rpm-client
 python setup.py sdist bdist_wheel --python-tag py3
-find . -name "*.whl" -exec pip install {} \;
+find . -name "*.whl" -exec docker exec pulp pip3 install /root/pulp-openapi-generator/pulp_rpm-client/{} \;
 tar cvf ../../pulp_rpm/python-client.tar ./dist
 
 find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \;

.github/workflows/scripts/script.sh

@@ -61,50 +61,34 @@ if [[ "$TEST" == "plugin-from-pypi" ]]; then
   git checkout ${COMPONENT_VERSION} -- pulp_rpm/tests/
 fi
 
+echo "machine pulp
+login admin
+password password
+" | cmd_stdin_prefix bash -c "cat > /root/.netrc"
+
+cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt"
+cat functest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt"
+cmd_prefix pip3 install -r /tmp/unittest_requirements.txt
+cmd_prefix pip3 install -r /tmp/functest_requirements.txt
+cmd_prefix pip3 install --upgrade ../pulp-smash
+
 cd ../pulp-openapi-generator
+./generate.sh pulp_rpm python
+cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_rpm-client
+sudo rm -rf ./pulp_rpm-client
 ./generate.sh pulpcore python
-pip install ./pulpcore-client
-rm -rf ./pulpcore-client
-if [[ "$TEST" = 'bindings' ]]; then
-  ./generate.sh pulpcore ruby 0
-  cd pulpcore-client
-  gem build pulpcore_client.gemspec
-  gem install --both ./pulpcore_client-0.gem
-fi
+cmd_prefix pip3 install /root/pulp-openapi-generator/pulpcore-client
+sudo rm -rf ./pulpcore-client
 ./generate.sh pulp_file python
-pip install ./pulp_file-client
-rm -rf ./pulp_file-client
-if [[ "$TEST" = 'bindings' ]]; then
-  ./generate.sh pulp_file ruby 0
-  cd pulp_file-client
-  gem build pulp_file_client.gemspec
-  gem install --both ./pulp_file_client-0.gem
-  cd ..
-fi
+cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_file-client
+sudo rm -rf ./pulp_file-client
 ./generate.sh pulp_certguard python
-pip install ./pulp_certguard-client
-rm -rf ./pulp_certguard-client
-if [[ "$TEST" = 'bindings' ]]; then
-  ./generate.sh pulp-certguard ruby 0
-  cd pulp-certguard-client
-  gem build pulp-certguard_client.gemspec
-  gem install --both ./pulp-certguard_client-0.gem
-  cd ..
-fi
+cmd_prefix pip3 install /root/pulp-openapi-generator/pulp_certguard-client
+sudo rm -rf ./pulp_certguard-client
 cd $REPO_ROOT
 
-if [[ "$TEST" = 'bindings' ]]; then
-  if [ -f $REPO_ROOT/.ci/assets/bindings/test_bindings.py ]; then
-    python $REPO_ROOT/.ci/assets/bindings/test_bindings.py
-  fi
-  if [ -f $REPO_ROOT/.ci/assets/bindings/test_bindings.rb ]; then
-    ruby $REPO_ROOT/.ci/assets/bindings/test_bindings.rb
-  fi
-  exit
-fi
-
-cat unittest_requirements.txt | cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt"
-cmd_prefix pip3 install -r /tmp/unittest_requirements.txt
+CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())')
+cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt  | tee -a "$CERTIFI" > /dev/null"
 
 # check for any uncommitted migrations
 echo "Checking for uncommitted migrations..."
@@ -116,15 +100,11 @@ if [[ "$TEST" != "upgrade" ]]; then
 fi
 
 # Run functional tests
-export PYTHONPATH=$REPO_ROOT/../pulp_file${PYTHONPATH:+:${PYTHONPATH}}
-export PYTHONPATH=$REPO_ROOT/../pulp-certguard${PYTHONPATH:+:${PYTHONPATH}}
-export PYTHONPATH=$REPO_ROOT${PYTHONPATH:+:${PYTHONPATH}}
-
 if [[ "$TEST" == "performance" ]]; then
   if [[ -z ${PERFORMANCE_TEST+x} ]]; then
-    pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance
+    cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance"
   else
-    pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance.test_$PERFORMANCE_TEST
+    cmd_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_rpm.tests.performance.test_$PERFORMANCE_TEST"
   fi
   exit
 fi
@@ -133,21 +113,21 @@ if [ -f $FUNC_TEST_SCRIPT ]; then
   source $FUNC_TEST_SCRIPT
 else
 
-    if [[ "$GITHUB_WORKFLOW" == "Rpm Nightly CI/CD" ]]; then
-        pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8  --nightly
-        pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m "not parallel"  --nightly
+    if [[ "$GITHUB_WORKFLOW" == "Rpm Nightly CI/CD" ]] || [[ "${RELEASE_WORKFLOW:-false}" == "true" ]]; then
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8 --nightly"
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel' --nightly"
 
 
-        pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and parallel" -n  8  --nightly
-        pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and not parallel"  --nightly
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n  8 --nightly"
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'  --nightly"
 
     else
-        pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8
-        pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m "not parallel"
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_rpm.tests.functional -m parallel -n 8"
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_rpm.tests.functional -m 'not parallel'"
 
 
-        pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and parallel" -n  8
-        pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m "from_pulpcore_for_all_plugins and not parallel"
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and parallel' -n  8"
+        cmd_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulpcore.tests.functional -m 'from_pulpcore_for_all_plugins and not parallel'"
 
     fi
 

requirements.txt

@@ -1,6 +1,6 @@
 createrepo_c~=0.20.0
 django_readonly_field~=1.1.1
-jsonschema~=4.6
+jsonschema~=4.6.0
 libcomps~=0.1.15
 productmd~=1.33.0
 pulpcore~=3.14.5