Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introducing github actions #182

Closed
wants to merge 1 commit into from
Closed

Introducing github actions #182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
113 changes: 113 additions & 0 deletions .github/before_install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
#!/usr/bin/env bash

set -mveuo pipefail

export PRE_BEFORE_INSTALL=$GITHUB_WORKSPACE/.github/pre_before_install.sh
export POST_BEFORE_INSTALL=$GITHUB_WORKSPACE/.github/post_before_install.sh

COMMIT_MSG=$(git log --format=%B --no-merges -1)
export COMMIT_MSG

if [ -f $PRE_BEFORE_INSTALL ]; then
$PRE_BEFORE_INSTALL
fi

if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*" | grep -v "https") ]]; then
echo "Invalid Required PR link detected in commit message. Please use the full https url."
exit 1
fi

if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ -z "${GITHUB_REF##*/}" -a "${GITHUB_REF##*/}" != "master" ]
then
export PULP_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulpcore\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_SMASH_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-smash\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_ROLES_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/ansible-pulp\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_BINDINGS_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-openapi-generator\/pull\/(\d+)' | awk -F'/' '{print $7}')
export PULP_OPERATOR_PR_NUMBER=$(echo $COMMIT_MSG | grep -oP 'Required\ PR:\ https\:\/\/github\.com\/pulp\/pulp-operator\/pull\/(\d+)' | awk -F'/' '{print $7}')
else
export PULP_PR_NUMBER=
export PULP_SMASH_PR_NUMBER=
export PULP_ROLES_PR_NUMBER=
export PULP_BINDINGS_PR_NUMBER=
export PULP_OPERATOR_PR_NUMBER=
fi

# dev_requirements contains tools needed for flake8, etc.
# So install them here rather than in install.sh
pip install -r dev_requirements.txt

cd ../pulp_catdog
# run black separately from flake8 to get a diff
black --diff --check --exclude docs/conf.py .

# Lint code.
#flake8 --config flake8.cfg

cd ..
git clone --depth=1 https://github.com/pulp/ansible-pulp.git
if [ -n "$PULP_ROLES_PR_NUMBER" ]; then
cd ansible-pulp
git fetch --depth=1 origin pull/$PULP_ROLES_PR_NUMBER/head:$PULP_ROLES_PR_NUMBER
git checkout $PULP_ROLES_PR_NUMBER
cd ..
fi


git clone --depth=1 https://github.com/pulp/pulp-operator.git
if [ -n "$PULP_OPERATOR_PR_NUMBER" ]; then
cd pulp-operator
git fetch --depth=1 origin pull/$PULP_OPERATOR_PR_NUMBER/head:$PULP_OPERATOR_PR_NUMBER
git checkout $PULP_OPERATOR_PR_NUMBER
RELEASE_VERSION=v0.9.0
curl -LO https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
chmod +x operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu && sudo mkdir -p /usr/local/bin/ && sudo cp operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu /usr/local/bin/operator-sdk && rm operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
sudo operator-sdk build --image-builder=docker quay.io/pulp/pulp-operator:latest
cd ..
fi

git clone https://github.com/pulp/pulp-openapi-generator.git
if [ -n "$PULP_BINDINGS_PR_NUMBER" ]; then
cd pulp-openapi-generator
git fetch origin +refs/pull/$PULP_BINDINGS_PR_NUMBER/merge
git checkout FETCH_HEAD
cd ..
fi

git clone --depth=1 https://github.com/pulp/pulpcore.git

if [ -n "$PULP_PR_NUMBER" ]; then
cd pulpcore
git fetch --depth=1 origin pull/$PULP_PR_NUMBER/head:$PULP_PR_NUMBER
git checkout $PULP_PR_NUMBER
cd ..
fi

rm ./pulpcore/containers/images/pulp/container-assets/pulp-api
mv ./plugin_template/.github/pulp-api ./pulpcore/containers/images/pulp/container-assets/pulp-api

# When building a (release) tag, we don't need the development modules for the
# build (they will be installed as dependencies of the plugin).
if [ -z "${GITHUB_REF##*/}" ]; then

git clone --depth=1 https://github.com/pulp/pulp-smash.git

if [ -n "$PULP_SMASH_PR_NUMBER" ]; then
cd pulp-smash
git fetch --depth=1 origin pull/$PULP_SMASH_PR_NUMBER/head:$PULP_SMASH_PR_NUMBER
git checkout $PULP_SMASH_PR_NUMBER
cd ..
fi

# pulp-smash already got installed via test_requirements.txt
pip install --upgrade --force-reinstall ./pulp-smash

fi


pip install ansible

cd pulp_catdog

if [ -f $POST_BEFORE_INSTALL ]; then
$POST_BEFORE_INSTALL
fi
7 changes: 7 additions & 0 deletions .github/func_test_script.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
# coding=utf-8

set -mveuo pipefail

cd ../pulp_catdog
pytest -v -r sx --color=yes --pyargs pulp_catdog.tests.functional || show_logs_and_return_non_zero
14 changes: 14 additions & 0 deletions .github/pre_before_install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/usr/bin/env bash

echo ::group::test_requirements
pip install -r test_requirements.txt
echo ::endgroup::
./plugin-template --generate-config --plugin-app-label catdog pulp_catdog
mkdir ../pulp_catdog/.github
mkdir ../pulp_catdog/.travis # due to test_bindings
touch ../pulp_catdog/.travis/test_bindings.py # due to test_bindings
echo 'pypi_username: the_pypi_user' >> ../pulp_catdog/template_config.yml
./plugin-template --all pulp_catdog
mv ../pulp_catdog/*.txt .
mv ../pulp_catdog/.github/before_install.sh ../pulp_catdog/.github/old_before_install.sh
find ../pulp_catdog/.github -type f -exec mv {} ./.github/ \;
27 changes: 27 additions & 0 deletions .github/pulp-api
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash -x

/usr/bin/wait_on_postgres.py

# Generating /var/lib/pulp/static at runtime rather than at container build time
# facilitates all of /var/lib/pulp being a separate volume.
django-admin collectstatic --noinput

# Get list of installed plugins via pip
# Assumes they are all named like "pulp-file" -> "file", with no 2nd dash.
# (Was previously needed when we ran `django-admin makemigrations`)
# PLUGINS=$(pip list | awk -F '[[:space:]]+|[-]' '/pulp-/{printf $2 " " }')

django-admin makemigrations catdog
django-admin migrate --noinput
django-admin migrate auth --noinput

if [ -n "${PULP_ADMIN_PASSWORD}" ]; then
django-admin reset-admin-password --password "${PULP_ADMIN_PASSWORD}"
fi

# NOTE: Due to the Linux dual-stack functionality, this will listen on both IPv4
# IPv6, even though netstat may seem to indicate it is IPv6 only.
# Due to containers using network namespaces, even if the host has this disabled
# with /proc/sys/net/ipv6/bindv6only=1, the container will still have
# it enabled with /proc/sys/net/ipv6/bindv6only=0 .
exec gunicorn --bind '[::]:24817' pulpcore.app.wsgi:application
76 changes: 76 additions & 0 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
name: Pulp CI
on: [push, pull_request]
jobs:
pulp:
runs-on: ubuntu-latest
env:
TEST: pulp
services:
postgres:
image: postgres:9.6
redis:
image: redis
steps:
- uses: actions/checkout@v2
- run: |
git fetch --prune --unshallow
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: Install httpie
run: |
echo ::group::HTTPIE
sudo apt-get update -yq
sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie
echo ::endgroup::
- name: Check commit message
if: github.event_name == 'pull_request'
env:
GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
run: |
echo ::group::REQUESTS
pip install requests
echo ::endgroup::
for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')
do
python .github/validate_commit_message.py $sha
VALUE=$?
if [ "$VALUE" -gt 0 ]; then
exit $VALUE
fi
done
shell: bash
- name: Before Install
run: |
.github/before_install.sh
echo "::set-env name=TEST::${{ matrix.env.TEST }}"
CURRENT_DIR=$PWD
cd ..
find ./ -type f -exec sed -i -e 's/timeout 90 bash/timeout 120 bash/g' {} \;
find ./ -type f -exec sed -i -e 's/\/usr\/local\/bin\/kubectl/$(which kubectl)/g' {} \;
find ./ -type f -exec sed -i -e 's/# containers are not running yet./sudo cp -i \/etc\/rancher\/k3s\/k3s.yaml $HOME\/.kube\/config/g' {} \;
cd $CURRENT_DIR
shell: bash
- name: Install
run: |
mkdir -p $HOME/.kube
.github/install.sh
- name: Before Script
run: |
.github/before_script.sh
echo "::set-env name=HTTPIE_CONFIG_DIR::$GITHUB_WORKSPACE/.github"
- name: Script
run: .github/script.sh
shell: bash
- name: After failure
if: failure()
run: |
http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/
sudo docker images
sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000
sudo kubectl logs -l name=pulp-operator -c operator --tail=10000
sudo kubectl logs -l app=pulp-api --tail=50000
sudo kubectl logs -l app=pulp-content --tail=10000
sudo kubectl logs -l app=pulp-resource-manager --tail=10000
sudo kubectl logs -l app=pulp-worker --tail=10000
60 changes: 60 additions & 0 deletions .github/workflows/cron.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
name: Cron jobs
on:
schedule:
# * is a special character in YAML so you have to quote this string
- cron: '15 1 * * *'
jobs:
pulp:
runs-on: ubuntu-latest
env:
TEST: pulp
services:
postgres:
image: postgres:9.6
redis:
image: redis
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: '3.7'
- name: Install httpie
run: |
echo ::group::HTTPIE
sudo apt-get update -yq
sudo -E apt-get -yq --no-install-suggests --no-install-recommends install httpie
echo ::endgroup::
- name: Before Install
run: |
.github/before_install.sh
echo "::set-env name=TEST::${{ matrix.env.TEST }}"
CURRENT_DIR=$PWD
cd ..
find ./ -type f -exec sed -i -e 's/timeout 90 bash/timeout 120 bash/g' {} \;
find ./ -type f -exec sed -i -e 's/\/usr\/local\/bin\/kubectl/$(which kubectl)/g' {} \;
find ./ -type f -exec sed -i -e 's/# containers are not running yet./sudo cp -i \/etc\/rancher\/k3s\/k3s.yaml $HOME\/.kube\/config/g' {} \;
cd $CURRENT_DIR
shell: bash
- name: Install
run: |
mkdir -p $HOME/.kube
.github/install.sh
- name: Before Script
run: |
.github/before_script.sh
echo "::set-env name=HTTPIE_CONFIG_DIR::$GITHUB_WORKSPACE/.github"
- name: Script
run: .github/script.sh
shell: bash
- name: After failure
if: failure()
run: |
http --timeout 30 --check-status --pretty format --print hb http://localhost:24817/pulp/api/v3/status/
sudo docker images
sudo kubectl logs -l name=pulp-operator -c ansible --tail=10000
sudo kubectl logs -l name=pulp-operator -c operator --tail=10000
sudo kubectl logs -l app=pulp-api --tail=50000
sudo kubectl logs -l app=pulp-content --tail=10000
sudo kubectl logs -l app=pulp-resource-manager --tail=10000
sudo kubectl logs -l app=pulp-worker --tail=10000
7 changes: 0 additions & 7 deletions .travis/before_install.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
#!/usr/bin/env bash

# WARNING: DO NOT EDIT!
#
# This file was generated by plugin_template, and is managed by bootstrap.py. Please use
# bootstrap.py to update this file.
#
# For more info visit https://github.com/pulp/plugin_template

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why should this header be removed?
The reference to bootstrap.py is wrong though.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not from the template, it is the CI config for plugin_template itself, and it has some differences from the one generated from template.

set -mveuo pipefail

export PRE_BEFORE_INSTALL=$TRAVIS_BUILD_DIR/.travis/pre_before_install.sh
Expand Down
7 changes: 6 additions & 1 deletion plugin-template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ def main():
help=textwrap.dedent('''\
Generate or update travis configuration files.

'''))
parser.add_argument('--github', action='store_true',
help=textwrap.dedent('''\
Generate or update github actions configuration files.

'''))
parser.add_argument('--docs', action='store_true',
help=textwrap.dedent('''\
Expand Down Expand Up @@ -155,7 +160,7 @@ def main():
print("If bindings are not excluded, `test_bindings.py` must exist.")
return 2

for section in ['generate_config', 'bootstrap', 'travis', 'docs', 'test']:
for section in ['generate_config', 'bootstrap', 'travis', 'github', 'docs', 'test']:
if getattr(args, section) or args.all:
write_template_section(config, section, plugin_root_dir, verbose=args.verbose)

Expand Down