-
Notifications
You must be signed in to change notification settings - Fork 35
/
script.sh.j2
executable file
·162 lines (137 loc) · 6.7 KB
/
script.sh.j2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
#!/usr/bin/env bash
# coding=utf-8
{% include 'header.j2' %}
set -mveuo pipefail
# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..
source .github/workflows/scripts/utils.sh
export POST_SCRIPT=$PWD/.github/workflows/scripts/post_script.sh
export FUNC_TEST_SCRIPT=$PWD/.github/workflows/scripts/func_test_script.sh
# Needed for starting the service
# Gets set in .github/settings.yml, but doesn't seem to inherited by
# this script.
export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py
export PULP_URL="{{ pulp_scheme }}://pulp"
REPORTED_STATUS="$(pulp status)"
echo "machine pulp
login admin
password password
" | cmd_user_stdin_prefix bash -c "cat >> ~pulp/.netrc"
# Some commands like ansible-galaxy specifically require 600
cmd_prefix bash -c "chmod 600 ~pulp/.netrc"
# Generate bindings
###################
echo "::group::Generate bindings"
touch bindings_requirements.txt
pushd ../pulp-openapi-generator
# Use app_label to generate api.json and package to produce the proper package name.
# Workaround: Domains are not supported by the published bindings.
# Sadly: Different pulpcore-versions aren't either...
# So we exclude the prebuilt ones only for domains disabled.
if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ]
then
BUILT_CLIENTS=""
else
BUILT_CLIENTS="{% for plugin in plugins %} {{ plugin.app_label }}{% endfor %} "
fi
for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}")
do
COMPONENT="$(jq -r '.component' <<<"${ITEM}")"
VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")"
# On older status endpoints, the module was not provided, but the package should be accurate
# there, because we did not merge plugins into pulpcore back then.
MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")"
PACKAGE="${MODULE%%.*}"
cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json"
if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]]
then
rm -rf "./${PACKAGE}-client"
./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}"
pushd "${PACKAGE}-client"
python setup.py sdist bdist_wheel --python-tag py3
popd
else
if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ]
then
ls -lR "${PACKAGE}-client/"
echo "Error: Client bindings for ${COMPONENT} not found."
echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing."
exit 1
fi
fi
echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../{{ plugin_name }}/bindings_requirements.txt"
done
popd
echo "::endgroup::"
echo "::group::Debug bindings diffs"
{% for plugin in plugins %}
# Bindings diff for {{ plugin.app_label }}
jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "{{ plugin.app_label }}-api.json" > "build-api.json"
jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/{{ plugin.app_label }}-api.json" > "test-api.json"
jsondiff --indent 2 build-api.json test-api.json || true
{%- endfor %}
echo "::endgroup::"
# Install test requirements
###########################
# Add a safeguard to make sure the proper versions of the clients are installed.
echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt
cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt
cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt
cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt
cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt
cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt
CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())')
cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'"
# check for any uncommitted migrations
echo "Checking for uncommitted migrations..."
{%- for plugin in plugins %}
cmd_user_prefix bash -c "django-admin makemigrations {{ plugin.app_label }} --check --dry-run"
{%- endfor %}
# Run unit tests.
{%- for plugin in plugins %}
cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs {{ plugin.name | snake }}.tests.unit"
{%- endfor %}
{%- if plugins %}
# Run functional tests
if [[ "$TEST" == "performance" ]]; then
if [[ -z ${PERFORMANCE_TEST+x} ]]; then
{%- for plugin in plugins %}
cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs {{ plugin.name | snake }}.tests.performance"
{%- endfor %}
else
{%- for plugin in plugins %}
cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs {{ plugin.name | snake }}.tests.performance.test_${PERFORMANCE_TEST}"
{%- endfor %}
fi
exit
fi
if [ -f "$FUNC_TEST_SCRIPT" ]; then
source "$FUNC_TEST_SCRIPT"
else
if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]]
then
{%- for plugin in plugins %}
cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs {{ plugin.name | snake }}.tests.functional -m parallel -n {{ parallel_test_workers }} --nightly"
cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs {{ plugin.name | snake }}.tests.functional -m 'not parallel' --nightly"
{%- endfor %}
else
{%- for plugin in plugins %}
cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs {{ plugin.name | snake }}.tests.functional -m parallel -n {{ parallel_test_workers }}"
cmd_user_prefix bash -c "pytest -v --timeout=300 -r sx --color=yes --suppress-no-test-exit-code --pyargs {{ plugin.name | snake }}.tests.functional -m 'not parallel'"
{%- endfor %}
fi
fi
{%- if test_cli %}
{%- if docker_fixtures %}
export PULP_FIXTURES_URL="http://pulp-fixtures:8080"
{%- endif %}
pushd ../{{ cli_package }}
pip install -r test_requirements.txt
pytest -v -m "{{ plugins | map(attribute='name') | join(' or ') | snake }}"
popd
{%- endif %}
{%- endif %}
if [ -f "$POST_SCRIPT" ]; then
source "$POST_SCRIPT"
fi