Skip to content

Commit 0624cf1

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
Merge pull request #304 from stackql/dev
new-testing-interface
2 parents 5af778f + 25f2497 commit 0624cf1

File tree

12 files changed

+208
-41
lines changed

12 files changed

+208
-41
lines changed

‎.github/workflows/regression.yml‎

Lines changed: 95 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,62 @@ on:
1515
- integration*
1616

1717
env:
18+
IS_TAG: ${{ github.ref_type == 'tag' }}
1819
GO_VERSION: '^1.22'
1920
STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }}
2021
STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }}
2122
STACKQL_ANY_SDK_REPOSITORY: ${{ vars.STACKQL_ANY_SDK_REPOSITORY != '' && vars.STACKQL_ANY_SDK_REPOSITORY || 'stackql/any-sdk' }}
2223
STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }}
2324

2425
jobs:
26+
27+
test_python_package_build:
28+
# id: test_python_package_build
29+
name: Test Python Package Build
30+
runs-on: ubuntu-22.04
31+
timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
32+
steps:
33+
34+
- name: Check out code into the Go module directory
35+
uses: actions/checkout@v4.1.1
36+
with:
37+
repository: ${{ env.STACKQL_CORE_REPOSITORY }}
38+
ref: ${{ env.STACKQL_CORE_REF }}
39+
token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
40+
path: stackql-core-pkg
41+
42+
- name: Setup Python
43+
uses: actions/setup-python@v5.0.0
44+
with:
45+
cache: pip
46+
python-version: '3.12'
47+
48+
- name: Install Poetry
49+
uses: snok/install-poetry@v1
50+
with:
51+
version: 1.8.3
52+
virtualenvs-create: true
53+
virtualenvs-in-project: false
54+
virtualenvs-path: stackql-core-pkg/my-custom-path
55+
installer-parallel: true
56+
57+
58+
- name: Build package
59+
working-directory: stackql-core-pkg
60+
run: |
61+
cicd/util/01-build-robot-lib.sh
62+
63+
- name: Upload python package artifact
64+
uses: actions/upload-artifact@v4.3.1
65+
with:
66+
name: python-package-dist-folder
67+
path: stackql-core-pkg/test/dist
68+
2569
regression-testing:
2670
name: regression-testing
2771
runs-on: ubuntu-latest
72+
needs: test_python_package_build
73+
timeout-minutes: ${{ vars.DEFAULT_JOB_TIMEOUT_MIN == '' && 120 || vars.DEFAULT_JOB_TIMEOUT_MIN }}
2874
permissions:
2975
id-token: write
3076
contents: read
@@ -72,7 +118,20 @@ jobs:
72118
- name: Setup Python
73119
uses: actions/setup-python@v5.0.0
74120
with:
75-
python-version: '3.11'
121+
python-version: '3.12'
122+
123+
- name: Download python package dist folder
124+
uses: actions/download-artifact@v4.1.2
125+
with:
126+
name: python-package-dist-folder
127+
path: test/dist
128+
129+
- name: Install python testing package
130+
run: |
131+
echo "Inspecting python package"
132+
for file in test/dist/*.whl; do
133+
pip3 install "$file" --force-reinstall
134+
done
76135
77136
- name: Add dependencies
78137
working-directory: stackql-core
@@ -115,15 +174,15 @@ jobs:
115174
- name: Generate rewritten registry for simulations
116175
working-directory: stackql-core
117176
run: |
118-
python3 test/python/registry-rewrite.py
177+
python3 test/python/stackql_test_tooling/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
119178
120179
121180
- name: Prepare load balancing materials
122181
working-directory: stackql-core
123182
run: |
124183
sudo cp /etc/hosts /etc/hosts.bak
125-
python3 test/python/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts
126-
python3 test/python/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf
184+
python3 test/python/stackql_test_tooling/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts
185+
python3 test/python/stackql_test_tooling/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf
127186
128187
129188
- name: Install and run nginx load balancer
@@ -152,8 +211,8 @@ jobs:
152211
working-directory: stackql-core
153212
run: |
154213
pgrep -f flask | xargs kill -9 || true
155-
flask --app=./test/python/flask/gcp/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 &
156-
flask --app=./test/python/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 &
214+
flask --app=./test/python/stackql_test_tooling/flask/gcp/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 &
215+
flask --app=./test/python/stackql_test_tooling/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 &
157216
158217
- name: Run any-sdk cli mocked testing
159218
working-directory: stackql-core
@@ -181,12 +240,22 @@ jobs:
181240
run: |
182241
providerRoot="$(realpath $(pwd)/../providers)"
183242
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
184-
robot \
185-
--variable "${sundryCfg}" \
186-
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
187-
--include registry \
188-
-d test/robot/reports \
189-
test/robot/functional || true
243+
# Only tag runs non blocking on fail, hard stop on fail for branch checks
244+
if [ "${{ env.IS_TAG }}" = "true" ]; then
245+
robot \
246+
--variable "${sundryCfg}" \
247+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
248+
--include registry \
249+
-d test/robot/reports \
250+
test/robot/functional || true
251+
else
252+
robot \
253+
--variable "${sundryCfg}" \
254+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
255+
--include registry \
256+
-d test/robot/reports \
257+
test/robot/functional
258+
fi
190259
191260
- name: Output from core proxied functional tests
192261
if: always()
@@ -211,11 +280,20 @@ jobs:
211280
run: |
212281
providerRoot="$(realpath $(pwd)/providers)"
213282
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
214-
robot \
215-
--variable "${sundryCfg}" \
216-
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
217-
-d test/robot/reports/mocked \
218-
test/robot/stackql/mocked || true
283+
# Only tag runs non blocking on fail, hard stop on fail for branch checks
284+
if [ "${{ env.IS_TAG }}" = "true" ]; then
285+
robot \
286+
--variable "${sundryCfg}" \
287+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
288+
-d test/robot/reports/mocked \
289+
test/robot/stackql/mocked || true
290+
else
291+
robot \
292+
--variable "${sundryCfg}" \
293+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
294+
-d test/robot/reports/mocked \
295+
test/robot/stackql/mocked
296+
fi
219297
220298
- name: Output from local registry mocked functional tests
221299
if: always()

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ stackql*.sh
77
stackql-zip
88
.stackql/
99
stackql-core/
10+
stackql-core-pkg/
1011
stackql-any-sdk/
1112
.venv/
1213
*.py[co]

‎docs/test-developer-guide.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
2+
# Test Developer Guide
3+
4+
5+
## Local Setup
6+
7+
You will need to have these on your machine:
8+
9+
- `python` at a version `>= 3.11`. To see if it is already present and what version, type this in a terminal `python --version`. Hopefully you will see something like `Python 3.13.2` appear. You can also try `python3 --version`; although if only this latter works then you will want to set up an alias (outside scope of this doc). If it is not present at all, then install it per [python downloads page](https://www.python.org/downloads/)
10+
- `go` at a version `>= 1.19`. To see if it is already present and what version, type this in a terminal `go version`. Hopefully you will see something like `go version go1.22.0 darwin/amd64` appear. If it is missing or some inadequate verison, then install it per [the official go install page](https://go.dev/doc/install).
11+
- `openssl`, prefererable version 3 sometging, although earlier versions will probably work. To see if it is already present and what version, type this in a terminal `openssl --version`. Hopefully you will see something like `OpenSSL 3.4.1 11 Feb 2025 (Library: OpenSSL 3.4.1 11 Feb 2025)` appear. If not present, then, on Mac, use [homebrew](https://brew.sh/) to install it with `brew install openssl`.
12+
13+
In addition, you will need shell scripts with exported credentials in the files (relative to repository root):
14+
15+
- `scripts/sec/sec-ro-stackql.sh`.
16+
- `scripts/sec/sec-rw-stackql.sh`.
17+
18+
Then, once all this is in place, test setup can be done as a "once-off" (run again when you want to update dependencies) with:
19+
20+
```bash
21+
22+
scripts/local/ci/01-gather.sh
23+
24+
scripts/local/ci/02-setup.sh
25+
26+
```
27+
28+
All sorts of stuff will happen here and may take a little while.
29+
30+
Then, to run readonly tests locally: `scripts/local/ci/03-run-live-readonly.sh`.
31+
32+
To run readwrite tests locally (more dangerous): `scripts/local/ci/04-run-live-readwrite.sh`.
33+
34+
35+
You can now develop new test cases. Once they are working locally, you can test them remotely (once you are in the appropriate `github` group) by pushing a `git` tag that begins
36+
with `robot`, `regression`, or `integration`.
37+
38+

‎scripts/local/ci/02-setup.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ if [ ! -d "${STACKQL_CORE_DIR}/.venv" ]; then
1515
>&2 echo "Virtual environment created."
1616
fi
1717

18-
source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
18+
source "${STACKQL_CORE_DIR}/.venv/bin/activate"
1919

2020
pip install -r "${STACKQL_CORE_DIR}/cicd/requirements.txt"
2121

@@ -25,7 +25,7 @@ cd "${STACKQL_CORE_DIR}"
2525

2626
python cicd/python/build.py --build
2727

28-
python test/python/registry-rewrite.py
28+
python test/python/stackql_test_tooling/registry_rewrite.py --srcdir "$(pwd)/test/registry/src" --destdir "$(pwd)/test/registry-mocked/src"
2929

3030
openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
3131
openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365

‎scripts/local/ci/04-run-live-readonly.sh‎ renamed to ‎scripts/local/ci/03-run-live-readonly.sh‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,16 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
66

77
REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
88

9-
_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-readwrite.sh"
9+
STACKQL_CORE_DIR="${REPOSITORY_ROOT_DIR}/stackql-core"
10+
11+
_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-rw-stackql.sh"
1012

1113
if [ -f "${_SEC_FILE}" ]; then
1214
source "${_SEC_FILE}"
1315
fi
1416

15-
if [ -f "scripts/sec/aws-ro-stackql.sh" ]; then
16-
source "scripts/sec/aws-ro-stackql.sh"
17+
if [ -f "scripts/sec/sec-ro-stackql.sh" ]; then
18+
source "scripts/sec/sec-ro-stackql.sh"
1719
fi
1820

1921
if [ "${AWS_ACCESS_KEY_ID}" = "" ]; then
@@ -33,7 +35,9 @@ fi
3335

3436
cd "${REPOSITORY_ROOT_DIR}"
3537

36-
source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
38+
source "${STACKQL_CORE_DIR}/.venv/bin/activate"
39+
40+
export PYTHONPATH="${PYTHONPATH}:${STACKQL_CORE_DIR}/test/python"
3741

3842
robot -d test/robot/reports/readonly test/robot/stackql/live/readonly
3943

‎scripts/local/ci/03-run-live-readwrite.sh‎ renamed to ‎scripts/local/ci/04-run-live-readwrite.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
66

77
REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
88

9-
_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-readwrite.sh"
9+
STACKQL_CORE_DIR="${REPOSITORY_ROOT_DIR}/stackql-core"
10+
11+
_SEC_FILE="${REPOSITORY_ROOT_DIR}/scripts/sec/sec-rw-stackql.sh"
1012

1113
if [ -f "${_SEC_FILE}" ]; then
1214
source "${_SEC_FILE}"
@@ -19,7 +21,9 @@ fi
1921

2022
cd "${REPOSITORY_ROOT_DIR}"
2123

22-
source "${REPOSITORY_ROOT_DIR}/.venv/bin/activate"
24+
source "${STACKQL_CORE_DIR}/.venv/bin/activate"
25+
26+
export PYTHONPATH="${PYTHONPATH}:${STACKQL_CORE_DIR}/test/python"
2327

2428
robot -d test/robot/reports/readwrite test/robot/stackql/live/readwrite
2529

‎scripts/local/ci/99-clean.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#! /usr/bin/env bash
2+
3+
>&2 echo "cleaning dependent repositories from local file system"
4+
5+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
6+
7+
REPOSITORY_ROOT_DIR="$(realpath ${SCRIPT_DIR}/../../..)"
8+
9+
cd "${REPOSITORY_ROOT_DIR}"
10+
11+
rm -rf stackql-core || true
12+
rm -rf stackql-any-sdk || true
13+

‎test/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dist/

‎test/robot/stackql/live/readonly/live_readonly.robot‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,21 @@ Simple Google Buckets List With Date Logic Contains Exemplifies Use of SQLite Ma
1818
... ${EMPTY}
1919
... Google-Buckets-List-With-Date-Logic-Contains-Exemplifies-Use-of-SQLite-Math-Functions
2020

21+
Simple Google IAM Service Accounts List
22+
Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" This is a valid case where the test is targetted at SQLite only
23+
[Tags] google iam service_accounts gooogle.iam google.iam.service_accounts tier_1
24+
${inputStr}= Catenate
25+
... select email
26+
... from google.iam.service_accounts
27+
... where projectsId = 'stackql-robot'
28+
... order by email desc
29+
... ;
30+
Stock Stackql Exec Inline Contains Both Streams
31+
... ${inputStr}
32+
... stackql\-robot\-rw\-sa@stackql\-robot.iam.gserviceaccount.com
33+
... ${EMPTY}
34+
... Google-Buckets-List-With-Date-Logic-Contains-Exemplifies-Use-of-SQLite-Math-Functions
35+
2136
AWS Route53 List Record Sets Simple
2237
[Documentation] It is fine for this to dump 404 infor to stderr. So long as the empty reusult is represented with a header row, all good.
2338
[Tags] aws route53 resource_record_sets aws.route53 aws.route53.resource_record_sets tier_1

‎test/robot/stackql/live/readonly/stackql.resource‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
*** Variables ***
22
${REPOSITORY_ROOT} ${CURDIR}${/}..${/}..${/}..${/}..${/}..
33
${CORE_REPOSITORY_ROOT} ${REPOSITORY_ROOT}${/}stackql-core
4-
${CORE_LIB_HOME} ${CORE_REPOSITORY_ROOT}${/}test${/}robot${/}lib
4+
${CORE_LIB_HOME} ${CORE_REPOSITORY_ROOT}${/}test${/}python${/}stackql_test_tooling
55
${EXECUTION_PLATFORM} native # to be overridden from command line, eg "docker"
66
${SQL_BACKEND} sqlite_embedded # to be overridden from command line, eg "postgres_tcp"
77
${IS_WSL} false # to be overridden from command line, with string "true"
@@ -17,19 +17,24 @@ ${SUNDRY_CONFIG}= SEPARATOR=
1717
... "AWS_RECORD_SET_REGION": "us-east-1",
1818
... "registry_path": "${CURDIR}${/}..${/}..${/}..${/}..${/}..${/}providers"
1919
... }
20+
${CWD_FOR_EXEC} ${CORE_REPOSITORY_ROOT} # works for self repository, can be overwritten when shared
21+
${WEB_SERVICE_LIBRARY} stackql_test_tooling.web_service_keywords
22+
${STACKQL_INTERFACE_LIBRARY} stackql_test_tooling.StackQLInterfaces
23+
${CLOUD_INTEGRATION_LIBRARY} stackql_test_tooling.CloudIntegration
2024

2125
*** Settings ***
2226
Library Process
2327
Library OperatingSystem
2428
# Variable first defined clobbers later defined therefore most specific variable file first.
2529
Variables ${CURDIR}${/}readonly_variables.py ${SUNDRY_CONFIG}
26-
Variables ${CORE_LIB_HOME}/stackql_context.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${USE_STACKQL_PREINSTALLED}
30+
Variables ${CORE_LIB_HOME}${/}stackql_context.py ${CORE_REPOSITORY_ROOT} ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${USE_STACKQL_PREINSTALLED}
2731
... ${SUNDRY_CONFIG}
2832
Library Process
2933
Library OperatingSystem
3034
Library String
31-
Library ${CORE_LIB_HOME}/StackQLInterfaces.py ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${CONCURRENCY_LIMIT}
32-
Library ${CORE_LIB_HOME}/CloudIntegration.py
35+
Library ${STACKQL_INTERFACE_LIBRARY} ${EXECUTION_PLATFORM} ${SQL_BACKEND} ${CONCURRENCY_LIMIT}
36+
Library ${CLOUD_INTEGRATION_LIBRARY}
37+
Library ${WEB_SERVICE_LIBRARY} ${CWD_FOR_EXEC}
3338

3439
*** Keywords ***
3540

0 commit comments

Comments
 (0)