From de19f9bd38d258f66677e37a166fd5242b84846d Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 18:42:59 +0300 Subject: [PATCH 01/10] PHP_CLI_SERVER_WORKERS=2 to `php -S` Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 8393482a..dd089f91 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -99,7 +99,7 @@ jobs: --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} php occ config:system:set loglevel --value=1 --type=integer php occ config:system:set debug --value=true --type=boolean - php -S localhost:8080 & + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - name: Checkout NcPyApi uses: actions/checkout@v4 @@ -243,7 +243,7 @@ jobs: --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} php occ config:system:set loglevel --value=1 php occ config:system:set debug --value=true --type=boolean - php -S localhost:8080 & + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - name: Checkout NcPyApi uses: actions/checkout@v4 @@ -381,7 +381,7 @@ jobs: --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} php occ config:system:set loglevel --value=1 --type=integer php occ config:system:set debug --value=true --type=boolean - php -S localhost:8080 & + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - name: Checkout NcPyApi uses: actions/checkout@v4 @@ -512,7 +512,7 @@ jobs: ./occ config:system:set debug --value=true --type=boolean ./occ app:enable notifications ./occ app:enable notes - php -S localhost:8080 & + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - name: Checkout NcPyApi uses: actions/checkout@v4 @@ -662,7 +662,7 @@ jobs: ./occ config:system:set debug --value=true --type=boolean ./occ app:enable notifications ./occ app:enable activity - php -S localhost:8080 & + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - name: Checkout NcPyApi uses: actions/checkout@v4 @@ -802,7 +802,7 @@ jobs: ./occ config:system:set debug --value=true --type=boolean ./occ app:enable activity ./occ app:enable notes - php -S localhost:8080 & + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - name: Checkout NcPyApi uses: actions/checkout@v4 From f6fb7c519df62c7028a5ca7197ad3afceba2882c Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 19:24:20 +0300 Subject: [PATCH 02/10] fixed test Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 2 +- nc_py_api/ex_app/integration_fastapi.py | 4 ++-- tests/_install_init_handler_models.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index dd089f91..0eb065fe 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -847,7 +847,7 @@ jobs: uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} - name: coverage_sqlite_${{ matrix.nextcloud }} + name: coverage_sqlite_${{ matrix.nextcloud }}_client file: coverage.xml fail_ci_if_error: true verbose: true diff --git a/nc_py_api/ex_app/integration_fastapi.py b/nc_py_api/ex_app/integration_fastapi.py index fc645b5e..ffb1e239 100644 --- a/nc_py_api/ex_app/integration_fastapi.py +++ b/nc_py_api/ex_app/integration_fastapi.py @@ -75,7 +75,7 @@ def set_handlers( :param models_download_params: Parameters to pass to ``snapshot_download`` function from **huggingface_hub**. """ - def fetch_models_task(models: dict): + def fetch_models_task(models: list[str]): if models: from huggingface_hub import snapshot_download # noqa isort:skip pylint: disable=C0415 disable=E0401 from tqdm import tqdm # noqa isort:skip pylint: disable=C0415 disable=E0401 @@ -115,5 +115,5 @@ def heartbeat_callback(): @fast_api_app.post("/init") def init_callback(background_tasks: BackgroundTasks): - background_tasks.add_task(fetch_models_task, models_to_fetch if models_to_fetch else {}) + background_tasks.add_task(fetch_models_task, models_to_fetch if models_to_fetch else []) return responses.JSONResponse(content={}, status_code=200) diff --git a/tests/_install_init_handler_models.py b/tests/_install_init_handler_models.py index 9d0f8ef0..7dda504a 100644 --- a/tests/_install_init_handler_models.py +++ b/tests/_install_init_handler_models.py @@ -10,7 +10,7 @@ def enabled_handler(_enabled: bool, _nc: NextcloudApp) -> str: if _enabled: try: - snapshot_download(MODEL_NAME, local_files_only=True) + snapshot_download(MODEL_NAME, local_files_only=True, cache_dir=ex_app.persistent_storage()) except Exception: # noqa return "model not found" return "" From a6ded79c5de6a0b6673019680ecfebb7e6b90653 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 20:43:27 +0300 Subject: [PATCH 03/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 253 ++++++++++++------------ tests/_install_init_handler_models.py | 4 +- tests/_tests_at_the_end.py | 23 --- 3 files changed, 134 insertions(+), 146 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 0eb065fe..66d12201 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -152,6 +152,17 @@ jobs: kill -15 $(cat /tmp/_talk_bot.pid) timeout 3m tail --pid=$(cat /tmp/_talk_bot.pid) -f /dev/null coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py + + ../php occ app_api:app:unregister "$APP_ID" --silent + ../php occ app_api:daemon:unregister manual_install + + coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & + echo $! > /tmp/_install_models.pid + python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 + sh scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" + kill -15 $(cat /tmp/_install_models.pid) + timeout 3m tail --pid=$(cat /tmp/_install_models.pid) -f /dev/null + coverage combine && coverage xml && coverage html - name: HTML coverage to artifacts @@ -323,127 +334,127 @@ jobs: path: data/nextcloud.log if-no-files-found: warn - tests-oci: - needs: [analysis] - runs-on: ubuntu-22.04 - name: ${{ matrix.nextcloud }} • 🐘${{ matrix.php-version }} • 🐍${{ matrix.python }} • OCI - strategy: - fail-fast: false - matrix: - nextcloud: [ "27.1.2" ] - python: [ "3.11" ] - php-version: [ "8.2" ] - - services: - oracle: - image: ghcr.io/gvenzl/oracle-xe:11 - env: - ORACLE_RANDOM_PASSWORD: true - APP_USER: useroracle - APP_USER_PASSWORD: userpassword - options: >- - --health-cmd healthcheck.sh - --health-interval 10s - --health-timeout 5s - --health-retries 10 - ports: - - 1521:1521/tcp - - steps: - - name: Set up php - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, \ - posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, oci8 - - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: cache-nextcloud - id: nextcloud_setup - uses: actions/cache@v3 - with: - path: nextcloud-${{ matrix.nextcloud }}.tar.bz2 - key: ${{ matrix.nextcloud }} - - - name: Download Nextcloud - if: steps.nextcloud_setup.outputs.cache-hit != 'true' - run: wget -q https://download.nextcloud.com/server/releases/nextcloud-${{ matrix.nextcloud }}.tar.bz2 - - - name: Set up Nextcloud - run: | - tar -xjf nextcloud-${{ matrix.nextcloud }}.tar.bz2 --strip-components 1 - mkdir data - php occ maintenance:install --verbose --database=oci --database-name=XE \ - --database-host=127.0.0.1 --database-port=1521 --database-user=useroracle --database-pass=userpassword \ - --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} - php occ config:system:set loglevel --value=1 --type=integer - php occ config:system:set debug --value=true --type=boolean - PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & - - - name: Checkout NcPyApi - uses: actions/checkout@v4 - with: - path: nc_py_api - - - name: Install NcPyApi - working-directory: nc_py_api - run: python3 -m pip -v install ".[app,dev-min]" - - - name: Checkout AppAPI - uses: actions/checkout@v4 - with: - path: apps/app_api - repository: cloud-py-api/app_api - - - name: Install AppAPI - run: | - php occ app:enable app_api - cd nc_py_api - coverage run --data-file=.coverage.ci_install tests/_install.py & - echo $! > /tmp/_install.pid - python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 - python3 tests/_app_security_checks.py http://127.0.0.1:$APP_PORT - cd .. - sh nc_py_api/scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" - kill -15 $(cat /tmp/_install.pid) - timeout 3m tail --pid=$(cat /tmp/_install.pid) -f /dev/null - - - name: Generate coverage report - working-directory: nc_py_api - run: | - coverage run --data-file=.coverage.ci -m pytest - coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py - coverage combine && coverage xml && coverage html - env: - SKIP_NC_CLIENT_TESTS: 1 - - - name: HTML coverage to artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} - path: nc_py_api/htmlcov - if-no-files-found: error - - - name: Upload report to Codecov - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} - file: coverage.xml - fail_ci_if_error: true - verbose: true - working-directory: nc_py_api - - - name: Upload NC logs - if: always() - uses: actions/upload-artifact@v3 - with: - name: nc_log_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} - path: data/nextcloud.log - if-no-files-found: warn +# tests-oci: +# needs: [analysis] +# runs-on: ubuntu-22.04 +# name: ${{ matrix.nextcloud }} • 🐘${{ matrix.php-version }} • 🐍${{ matrix.python }} • OCI +# strategy: +# fail-fast: false +# matrix: +# nextcloud: [ "27.1.2" ] +# python: [ "3.11" ] +# php-version: [ "8.2" ] +# +# services: +# oracle: +# image: ghcr.io/gvenzl/oracle-xe:11 +# env: +# ORACLE_RANDOM_PASSWORD: true +# APP_USER: useroracle +# APP_USER_PASSWORD: userpassword +# options: >- +# --health-cmd healthcheck.sh +# --health-interval 10s +# --health-timeout 5s +# --health-retries 10 +# ports: +# - 1521:1521/tcp +# +# steps: +# - name: Set up php +# uses: shivammathur/setup-php@v2 +# with: +# php-version: ${{ matrix.php-version }} +# extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, \ +# posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, oci8 +# +# - uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python }} +# +# - name: cache-nextcloud +# id: nextcloud_setup +# uses: actions/cache@v3 +# with: +# path: nextcloud-${{ matrix.nextcloud }}.tar.bz2 +# key: ${{ matrix.nextcloud }} +# +# - name: Download Nextcloud +# if: steps.nextcloud_setup.outputs.cache-hit != 'true' +# run: wget -q https://download.nextcloud.com/server/releases/nextcloud-${{ matrix.nextcloud }}.tar.bz2 +# +# - name: Set up Nextcloud +# run: | +# tar -xjf nextcloud-${{ matrix.nextcloud }}.tar.bz2 --strip-components 1 +# mkdir data +# php occ maintenance:install --verbose --database=oci --database-name=XE \ +# --database-host=127.0.0.1 --database-port=1521 --database-user=useroracle --database-pass=userpassword \ +# --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} +# php occ config:system:set loglevel --value=1 --type=integer +# php occ config:system:set debug --value=true --type=boolean +# PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & +# +# - name: Checkout NcPyApi +# uses: actions/checkout@v4 +# with: +# path: nc_py_api +# +# - name: Install NcPyApi +# working-directory: nc_py_api +# run: python3 -m pip -v install ".[app,dev-min]" +# +# - name: Checkout AppAPI +# uses: actions/checkout@v4 +# with: +# path: apps/app_api +# repository: cloud-py-api/app_api +# +# - name: Install AppAPI +# run: | +# php occ app:enable app_api +# cd nc_py_api +# coverage run --data-file=.coverage.ci_install tests/_install.py & +# echo $! > /tmp/_install.pid +# python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 +# python3 tests/_app_security_checks.py http://127.0.0.1:$APP_PORT +# cd .. +# sh nc_py_api/scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" +# kill -15 $(cat /tmp/_install.pid) +# timeout 3m tail --pid=$(cat /tmp/_install.pid) -f /dev/null +# +# - name: Generate coverage report +# working-directory: nc_py_api +# run: | +# coverage run --data-file=.coverage.ci -m pytest +# coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py +# coverage combine && coverage xml && coverage html +# env: +# SKIP_NC_CLIENT_TESTS: 1 +# +# - name: HTML coverage to artifacts +# uses: actions/upload-artifact@v3 +# with: +# name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} +# path: nc_py_api/htmlcov +# if-no-files-found: error +# +# - name: Upload report to Codecov +# uses: codecov/codecov-action@v3 +# with: +# token: ${{ secrets.CODECOV_TOKEN }} +# name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} +# file: coverage.xml +# fail_ci_if_error: true +# verbose: true +# working-directory: nc_py_api +# +# - name: Upload NC logs +# if: always() +# uses: actions/upload-artifact@v3 +# with: +# name: nc_log_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} +# path: data/nextcloud.log +# if-no-files-found: warn tests-latest-maria: needs: [analysis] diff --git a/tests/_install_init_handler_models.py b/tests/_install_init_handler_models.py index 7dda504a..ac957331 100644 --- a/tests/_install_init_handler_models.py +++ b/tests/_install_init_handler_models.py @@ -7,8 +7,8 @@ MODEL_NAME = "MBZUAI/LaMini-T5-61M" -def enabled_handler(_enabled: bool, _nc: NextcloudApp) -> str: - if _enabled: +def enabled_handler(enabled: bool, _nc: NextcloudApp) -> str: + if enabled: try: snapshot_download(MODEL_NAME, local_files_only=True, cache_dir=ex_app.persistent_storage()) except Exception: # noqa diff --git a/tests/_tests_at_the_end.py b/tests/_tests_at_the_end.py index 0631678b..04cec84d 100644 --- a/tests/_tests_at_the_end.py +++ b/tests/_tests_at_the_end.py @@ -28,26 +28,3 @@ def test_ex_app_enable_disable(nc_client, nc_app): assert nc_client.apps.ex_app_is_enabled("nc_py_api") is True finally: r.terminate() - - -def test_install_init_handler_models(nc_client, nc_app): - child_environment = os.environ.copy() - child_environment["APP_PORT"] = os.environ.get("APP_PORT", "9009") - r = Popen( - [ - sys.executable, - os.path.join(os.path.dirname(os.path.abspath(__file__)), "_install_init_handler_models.py"), - ], - env=child_environment, - cwd=os.getcwd(), - ) - url = f"http://127.0.0.1:{child_environment['APP_PORT']}/heartbeat" - try: - if check_heartbeat(url, '"status":"ok"', 15, 0.3): - raise RuntimeError("`_install_init_handler_models` can not start.") - if nc_client.apps.ex_app_is_enabled("nc_py_api"): - nc_client.apps.ex_app_disable("nc_py_api") - nc_client.apps.ex_app_enable("nc_py_api") - assert nc_client.apps.ex_app_is_enabled("nc_py_api") is True - finally: - r.terminate() From 4b919fafd375a2f0389ff997ca6fff5508f811d9 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 20:44:20 +0300 Subject: [PATCH 04/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 66d12201..61127751 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -876,7 +876,7 @@ jobs: permissions: contents: none runs-on: ubuntu-22.04 - needs: [tests-maria, tests-pgsql, tests-oci, tests-latest-maria, test-latest-pgsql, tests-client-sqlite] + needs: [tests-maria, tests-pgsql, tests-latest-maria, test-latest-pgsql, tests-client-sqlite] name: Tests-OK steps: - run: echo "Tests passed successfully" From 0735f5cc895d23699e545c298cbcafdca0fb584d Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 20:55:38 +0300 Subject: [PATCH 05/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 61127751..6e3be980 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -153,8 +153,8 @@ jobs: timeout 3m tail --pid=$(cat /tmp/_talk_bot.pid) -f /dev/null coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py - ../php occ app_api:app:unregister "$APP_ID" --silent - ../php occ app_api:daemon:unregister manual_install + php ../occ app_api:app:unregister "$APP_ID" --silent + php ../occ app_api:daemon:unregister manual_install coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & echo $! > /tmp/_install_models.pid From ddba2b079c9c47c90b02fe458ed058537ac98fd0 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 21:07:05 +0300 Subject: [PATCH 06/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 6e3be980..a3691ea7 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -153,8 +153,10 @@ jobs: timeout 3m tail --pid=$(cat /tmp/_talk_bot.pid) -f /dev/null coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py - php ../occ app_api:app:unregister "$APP_ID" --silent - php ../occ app_api:daemon:unregister manual_install + cd .. + php occ app_api:app:unregister "$APP_ID" --silent + php occ app_api:daemon:unregister manual_install + cd nc_py_api coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & echo $! > /tmp/_install_models.pid From 002e91ae5c1380e42c6852e7c86ab1537bdf930f Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 21:24:40 +0300 Subject: [PATCH 07/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index a3691ea7..82c8ef05 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -144,27 +144,24 @@ jobs: run: php occ app:enable spreed - name: Generate coverage report - working-directory: nc_py_api run: | + cd nc_py_api coverage run --data-file=.coverage.talk_bot tests/_talk_bot.py & echo $! > /tmp/_talk_bot.pid coverage run --data-file=.coverage.ci -m pytest kill -15 $(cat /tmp/_talk_bot.pid) timeout 3m tail --pid=$(cat /tmp/_talk_bot.pid) -f /dev/null coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py - cd .. php occ app_api:app:unregister "$APP_ID" --silent php occ app_api:daemon:unregister manual_install cd nc_py_api - coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & echo $! > /tmp/_install_models.pid python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 sh scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" kill -15 $(cat /tmp/_install_models.pid) timeout 3m tail --pid=$(cat /tmp/_install_models.pid) -f /dev/null - coverage combine && coverage xml && coverage html - name: HTML coverage to artifacts From c4cfe0b95eb6e6409f6c04d84b5e672e00648627 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 23:02:00 +0300 Subject: [PATCH 08/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 82c8ef05..01f5bb4c 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -143,19 +143,24 @@ jobs: - name: Enable Talk run: php occ app:enable spreed - - name: Generate coverage report + - name: Generate coverage report (1) + working-directory: nc_py_api run: | - cd nc_py_api coverage run --data-file=.coverage.talk_bot tests/_talk_bot.py & echo $! > /tmp/_talk_bot.pid coverage run --data-file=.coverage.ci -m pytest kill -15 $(cat /tmp/_talk_bot.pid) timeout 3m tail --pid=$(cat /tmp/_talk_bot.pid) -f /dev/null coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py - cd .. + + - name: Uninstall NcPyApi + run: | php occ app_api:app:unregister "$APP_ID" --silent php occ app_api:daemon:unregister manual_install - cd nc_py_api + + - name: Generate coverage report (2) + working-directory: nc_py_api + run: | coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & echo $! > /tmp/_install_models.pid python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 From 1568cafb666cb6adf50e311ea51576ba8bacd264 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sat, 21 Oct 2023 23:28:24 +0300 Subject: [PATCH 09/10] dev: tests fix Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 01f5bb4c..998d6632 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -164,9 +164,11 @@ jobs: coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & echo $! > /tmp/_install_models.pid python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 - sh scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" + cd .. + sh nc_py_api/scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" kill -15 $(cat /tmp/_install_models.pid) timeout 3m tail --pid=$(cat /tmp/_install_models.pid) -f /dev/null + cd nc_py_api coverage combine && coverage xml && coverage html - name: HTML coverage to artifacts From fbad3b301a525a48e2321bb7a2a3d8ce505eeb0f Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 22 Oct 2023 00:19:20 +0300 Subject: [PATCH 10/10] fixed tests Signed-off-by: Alexander Piskun --- .github/workflows/analysis-coverage.yml | 263 +++++++++++++----------- nc_py_api/ex_app/integration_fastapi.py | 2 +- tests/_install.py | 6 +- tests/_install_init_handler_models.py | 6 +- 4 files changed, 147 insertions(+), 130 deletions(-) diff --git a/.github/workflows/analysis-coverage.yml b/.github/workflows/analysis-coverage.yml index 998d6632..c1a3f281 100644 --- a/.github/workflows/analysis-coverage.yml +++ b/.github/workflows/analysis-coverage.yml @@ -304,7 +304,7 @@ jobs: - name: Enable Talk run: php occ app:enable spreed - - name: Generate coverage report + - name: Generate coverage report (1) working-directory: nc_py_api run: | coverage run --data-file=.coverage.talk_bot tests/_talk_bot.py & @@ -313,6 +313,23 @@ jobs: kill -15 $(cat /tmp/_talk_bot.pid) timeout 3m tail --pid=$(cat /tmp/_talk_bot.pid) -f /dev/null coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py + + - name: Uninstall NcPyApi + run: | + php occ app_api:app:unregister "$APP_ID" --silent + php occ app_api:daemon:unregister manual_install + + - name: Generate coverage report (2) + working-directory: nc_py_api + run: | + coverage run --data-file=.coverage.ci_install_models tests/_install_init_handler_models.py & + echo $! > /tmp/_install_models.pid + python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 + cd .. + sh nc_py_api/scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" + kill -15 $(cat /tmp/_install_models.pid) + timeout 3m tail --pid=$(cat /tmp/_install_models.pid) -f /dev/null + cd nc_py_api coverage combine && coverage xml && coverage html - name: HTML coverage to artifacts @@ -340,127 +357,127 @@ jobs: path: data/nextcloud.log if-no-files-found: warn -# tests-oci: -# needs: [analysis] -# runs-on: ubuntu-22.04 -# name: ${{ matrix.nextcloud }} • 🐘${{ matrix.php-version }} • 🐍${{ matrix.python }} • OCI -# strategy: -# fail-fast: false -# matrix: -# nextcloud: [ "27.1.2" ] -# python: [ "3.11" ] -# php-version: [ "8.2" ] -# -# services: -# oracle: -# image: ghcr.io/gvenzl/oracle-xe:11 -# env: -# ORACLE_RANDOM_PASSWORD: true -# APP_USER: useroracle -# APP_USER_PASSWORD: userpassword -# options: >- -# --health-cmd healthcheck.sh -# --health-interval 10s -# --health-timeout 5s -# --health-retries 10 -# ports: -# - 1521:1521/tcp -# -# steps: -# - name: Set up php -# uses: shivammathur/setup-php@v2 -# with: -# php-version: ${{ matrix.php-version }} -# extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, \ -# posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, oci8 -# -# - uses: actions/setup-python@v4 -# with: -# python-version: ${{ matrix.python }} -# -# - name: cache-nextcloud -# id: nextcloud_setup -# uses: actions/cache@v3 -# with: -# path: nextcloud-${{ matrix.nextcloud }}.tar.bz2 -# key: ${{ matrix.nextcloud }} -# -# - name: Download Nextcloud -# if: steps.nextcloud_setup.outputs.cache-hit != 'true' -# run: wget -q https://download.nextcloud.com/server/releases/nextcloud-${{ matrix.nextcloud }}.tar.bz2 -# -# - name: Set up Nextcloud -# run: | -# tar -xjf nextcloud-${{ matrix.nextcloud }}.tar.bz2 --strip-components 1 -# mkdir data -# php occ maintenance:install --verbose --database=oci --database-name=XE \ -# --database-host=127.0.0.1 --database-port=1521 --database-user=useroracle --database-pass=userpassword \ -# --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} -# php occ config:system:set loglevel --value=1 --type=integer -# php occ config:system:set debug --value=true --type=boolean -# PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & -# -# - name: Checkout NcPyApi -# uses: actions/checkout@v4 -# with: -# path: nc_py_api -# -# - name: Install NcPyApi -# working-directory: nc_py_api -# run: python3 -m pip -v install ".[app,dev-min]" -# -# - name: Checkout AppAPI -# uses: actions/checkout@v4 -# with: -# path: apps/app_api -# repository: cloud-py-api/app_api -# -# - name: Install AppAPI -# run: | -# php occ app:enable app_api -# cd nc_py_api -# coverage run --data-file=.coverage.ci_install tests/_install.py & -# echo $! > /tmp/_install.pid -# python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 -# python3 tests/_app_security_checks.py http://127.0.0.1:$APP_PORT -# cd .. -# sh nc_py_api/scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" -# kill -15 $(cat /tmp/_install.pid) -# timeout 3m tail --pid=$(cat /tmp/_install.pid) -f /dev/null -# -# - name: Generate coverage report -# working-directory: nc_py_api -# run: | -# coverage run --data-file=.coverage.ci -m pytest -# coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py -# coverage combine && coverage xml && coverage html -# env: -# SKIP_NC_CLIENT_TESTS: 1 -# -# - name: HTML coverage to artifacts -# uses: actions/upload-artifact@v3 -# with: -# name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} -# path: nc_py_api/htmlcov -# if-no-files-found: error -# -# - name: Upload report to Codecov -# uses: codecov/codecov-action@v3 -# with: -# token: ${{ secrets.CODECOV_TOKEN }} -# name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} -# file: coverage.xml -# fail_ci_if_error: true -# verbose: true -# working-directory: nc_py_api -# -# - name: Upload NC logs -# if: always() -# uses: actions/upload-artifact@v3 -# with: -# name: nc_log_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} -# path: data/nextcloud.log -# if-no-files-found: warn + tests-oci: + needs: [analysis] + runs-on: ubuntu-22.04 + name: ${{ matrix.nextcloud }} • 🐘${{ matrix.php-version }} • 🐍${{ matrix.python }} • OCI + strategy: + fail-fast: false + matrix: + nextcloud: [ "27.1.2" ] + python: [ "3.11" ] + php-version: [ "8.2" ] + + services: + oracle: + image: ghcr.io/gvenzl/oracle-xe:11 + env: + ORACLE_RANDOM_PASSWORD: true + APP_USER: useroracle + APP_USER_PASSWORD: userpassword + options: >- + --health-cmd healthcheck.sh + --health-interval 10s + --health-timeout 5s + --health-retries 10 + ports: + - 1521:1521/tcp + + steps: + - name: Set up php + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-version }} + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, \ + posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, oci8 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: cache-nextcloud + id: nextcloud_setup + uses: actions/cache@v3 + with: + path: nextcloud-${{ matrix.nextcloud }}.tar.bz2 + key: ${{ matrix.nextcloud }} + + - name: Download Nextcloud + if: steps.nextcloud_setup.outputs.cache-hit != 'true' + run: wget -q https://download.nextcloud.com/server/releases/nextcloud-${{ matrix.nextcloud }}.tar.bz2 + + - name: Set up Nextcloud + run: | + tar -xjf nextcloud-${{ matrix.nextcloud }}.tar.bz2 --strip-components 1 + mkdir data + php occ maintenance:install --verbose --database=oci --database-name=XE \ + --database-host=127.0.0.1 --database-port=1521 --database-user=useroracle --database-pass=userpassword \ + --admin-user admin --admin-pass ${{ env.NC_AUTH_PASS }} + php occ config:system:set loglevel --value=1 --type=integer + php occ config:system:set debug --value=true --type=boolean + PHP_CLI_SERVER_WORKERS=2 php -S localhost:8080 & + + - name: Checkout NcPyApi + uses: actions/checkout@v4 + with: + path: nc_py_api + + - name: Install NcPyApi + working-directory: nc_py_api + run: python3 -m pip -v install ".[app,dev-min]" + + - name: Checkout AppAPI + uses: actions/checkout@v4 + with: + path: apps/app_api + repository: cloud-py-api/app_api + + - name: Install AppAPI + run: | + php occ app:enable app_api + cd nc_py_api + coverage run --data-file=.coverage.ci_install tests/_install.py & + echo $! > /tmp/_install.pid + python3 tests/_install_wait.py http://127.0.0.1:$APP_PORT/heartbeat "\"status\":\"ok\"" 15 0.5 + python3 tests/_app_security_checks.py http://127.0.0.1:$APP_PORT + cd .. + sh nc_py_api/scripts/ci_register.sh "$APP_ID" "$APP_VERSION" "$APP_SECRET" "localhost" "$APP_PORT" + kill -15 $(cat /tmp/_install.pid) + timeout 3m tail --pid=$(cat /tmp/_install.pid) -f /dev/null + + - name: Generate coverage report + working-directory: nc_py_api + run: | + coverage run --data-file=.coverage.ci -m pytest + coverage run --data-file=.coverage.at_the_end -m pytest tests/_tests_at_the_end.py + coverage combine && coverage xml && coverage html + env: + SKIP_NC_CLIENT_TESTS: 1 + + - name: HTML coverage to artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} + path: nc_py_api/htmlcov + if-no-files-found: error + + - name: Upload report to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + name: coverage_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} + file: coverage.xml + fail_ci_if_error: true + verbose: true + working-directory: nc_py_api + + - name: Upload NC logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: nc_log_oci_${{ matrix.nextcloud }}_${{ matrix.python }}_${{ matrix.php-version }} + path: data/nextcloud.log + if-no-files-found: warn tests-latest-maria: needs: [analysis] @@ -882,7 +899,7 @@ jobs: permissions: contents: none runs-on: ubuntu-22.04 - needs: [tests-maria, tests-pgsql, tests-latest-maria, test-latest-pgsql, tests-client-sqlite] + needs: [tests-maria, tests-pgsql, tests-oci, tests-latest-maria, test-latest-pgsql, tests-client-sqlite] name: Tests-OK steps: - run: echo "Tests passed successfully" diff --git a/nc_py_api/ex_app/integration_fastapi.py b/nc_py_api/ex_app/integration_fastapi.py index ffb1e239..52acad47 100644 --- a/nc_py_api/ex_app/integration_fastapi.py +++ b/nc_py_api/ex_app/integration_fastapi.py @@ -75,7 +75,7 @@ def set_handlers( :param models_download_params: Parameters to pass to ``snapshot_download`` function from **huggingface_hub**. """ - def fetch_models_task(models: list[str]): + def fetch_models_task(models: list[str]) -> None: if models: from huggingface_hub import snapshot_download # noqa isort:skip pylint: disable=C0415 disable=E0401 from tqdm import tqdm # noqa isort:skip pylint: disable=C0415 disable=E0401 diff --git a/tests/_install.py b/tests/_install.py index 1cc58db1..746ca5b9 100644 --- a/tests/_install.py +++ b/tests/_install.py @@ -27,13 +27,17 @@ def enabled_handler(enabled: bool, nc: NextcloudApp) -> str: return "" +def init_handler(): + NextcloudApp().set_init_status(100) + + def heartbeat_callback(): return "ok" @APP.on_event("startup") def initialization(): - ex_app.set_handlers(APP, enabled_handler, heartbeat_callback) + ex_app.set_handlers(APP, enabled_handler, heartbeat_callback, init_handler=init_handler) if __name__ == "__main__": diff --git a/tests/_install_init_handler_models.py b/tests/_install_init_handler_models.py index ac957331..c2f2f66f 100644 --- a/tests/_install_init_handler_models.py +++ b/tests/_install_init_handler_models.py @@ -16,13 +16,9 @@ def enabled_handler(enabled: bool, _nc: NextcloudApp) -> str: return "" -def init_handler(): - NextcloudApp().set_init_status(100) - - @APP.on_event("startup") def initialization(): - ex_app.set_handlers(APP, enabled_handler, init_handler=init_handler, models_to_fetch=[MODEL_NAME]) + ex_app.set_handlers(APP, enabled_handler, models_to_fetch=[MODEL_NAME]) if __name__ == "__main__":