From 5314b12781a90f5390ace85d3d2b88f09a27e3ad Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 11:56:21 +0200 Subject: [PATCH 01/14] setup airflow e2e execution in workflow --- .github/workflows/integrations.yml | 1221 +++++++++++++++------------- 1 file changed, 634 insertions(+), 587 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 523702aa7..d8dd679f8 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -5,6 +5,7 @@ on: workflow_dispatch: schedule: - cron: "0 6 * * *" # Run every day at arbitrary time (6:00 AM UTC) + push: env: WORKSPACE_NAME: e2e-tests @@ -12,587 +13,633 @@ env: SLACK_WEBHOOK_URL: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} jobs: - fastai: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.10" ] - os: [ ubuntu-latest, windows-latest ] - include: - - python-version: "3.7.16" - os: macos-latest - - python-version: "3.10" - os: macos-latest - runs-on: ${{ matrix.os }} - name: 'fastai (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-fastai/.github/actions/e2e@master - with: - working_directory: neptune-fastai - - fastai-notify: - needs: [fastai] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.fastai.result }} - - kedro: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.8", "3.10" ] - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - name: 'kedro (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/kedro-neptune/.github/actions/e2e@main - with: - working_directory: kedro-neptune - - kedro-notify: - needs: [kedro] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.kedro.result }} - - prophet: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.10" ] - os: [ ubuntu-latest ] - include: - - python-version: "3.7.16" - os: macos-latest - - python-version: "3.10" - os: macos-latest - runs-on: ${{ matrix.os }} - name: 'prophet (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-prophet/.github/actions/e2e@main - with: - working_directory: neptune-prophet - - prophet-notify: - needs: [prophet] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.prophet.result }} - - keras: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.10" ] - os: [ ubuntu-latest, windows-latest ] - include: - - python-version: "3.7.16" - os: macos-latest - - python-version: "3.10" - os: macos-latest - runs-on: ${{ matrix.os }} - name: 'keras (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-tensorflow-keras/.github/actions/e2e@master - with: - working_directory: neptune-tensorflow-keras - - keras-notify: - needs: [keras] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.keras.result }} - - lightning: - env: - BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} - USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} - ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} - ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} - SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.10" ] - os: [ ubuntu-latest ] - include: - - python-version: "3.7.16" - os: macos-latest - - python-version: "3.10" - os: macos-latest - runs-on: ${{ matrix.os }} - name: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Test - uses: ./.github/actions/test-e2e - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - module: lightning - report_job: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' - - lightning-notify: - needs: [lightning] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.lightning.result }} - - huggingface: - env: - BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} - USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} - ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} - ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} - SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.10" ] - os: [ ubuntu-latest ] - include: - - python-version: "3.7.16" - os: macos-latest - - python-version: "3.10" - os: macos-latest - runs-on: ${{ matrix.os }} - name: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Test - uses: ./.github/actions/test-e2e - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - module: huggingface - report_job: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' - - huggingface-notify: - needs: [huggingface] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.huggingface.result }} - - zenml: - env: - BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} - USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} - ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} - ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} - SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} - ZENML_ANALYTICS_OPT_IN: false - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.8" ] - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - name: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Test - uses: ./.github/actions/test-e2e - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - module: zenml - report_job: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' - - zenml-notify: - needs: [zenml] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.zenml.result }} - - detectron2: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.8", "3.9" ] - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - name: 'detectron2 (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-detectron2/.github/actions/e2e@main - with: - working_directory: neptune-detectron2 - - detectron2-notify: - needs: [ detectron2 ] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.detectron2.result }} - - rlang: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - name: 'R (${{ matrix.os }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - os: ${{ matrix.os }} - python-version: "3.10" - - - name: Test integration - uses: neptune-ai/neptune-r/.github/actions/e2e@master - with: - working_directory: rlang - - rlang-notify: - needs: [ rlang ] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.rlang.result }} - - sklearn: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.9" ] - os: [ ubuntu-latest, macos-latest ] - runs-on: ${{ matrix.os }} - name: 'sklearn (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-sklearn/.github/actions/e2e@master - with: - working_directory: neptune-sklearn - - sklearn-notify: - needs: [ sklearn ] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.sklearn.result }} - - xgboost: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.9" ] - os: [ ubuntu-latest, macos-latest, windows-latest ] - runs-on: ${{ matrix.os }} - name: 'xgboost (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-xgboost/.github/actions/e2e@master - with: - working_directory: neptune-xgboost - - xgboost-notify: - needs: [ xgboost ] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.xgboost.result }} - - sacred: - env: - NEPTUNE_PROJECT: e2e-tests/integrations - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - python-version: [ "3.7", "3.10" ] - os: [ ubuntu-latest, macos-latest, windows-latest ] - runs-on: ${{ matrix.os }} - name: 'sacred (${{ matrix.os }} - py${{ matrix.python-version }})' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Install package - uses: ./.github/actions/install-package - with: - python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - - - name: Test integration - uses: neptune-ai/neptune-sacred/.github/actions/e2e@main - with: - working_directory: neptune-sacred - - sacred-notify: - needs: [ sacred ] - runs-on: ubuntu-latest - if: (success() || failure()) && github.ref == 'refs/heads/master' - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - - name: Notify - uses: ./.github/actions/job-notify - with: - slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.sacred.result }} - - mlflow: +# fastai: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.10" ] +# os: [ ubuntu-latest, windows-latest ] +# include: +# - python-version: "3.7.16" +# os: macos-latest +# - python-version: "3.10" +# os: macos-latest +# runs-on: ${{ matrix.os }} +# name: 'fastai (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-fastai/.github/actions/e2e@master +# with: +# working_directory: neptune-fastai +# +# fastai-notify: +# needs: [fastai] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.fastai.result }} +# +# kedro: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.8", "3.10" ] +# os: [ ubuntu-latest, macos-latest ] +# runs-on: ${{ matrix.os }} +# name: 'kedro (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/kedro-neptune/.github/actions/e2e@main +# with: +# working_directory: kedro-neptune +# +# kedro-notify: +# needs: [kedro] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.kedro.result }} +# +# prophet: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.10" ] +# os: [ ubuntu-latest ] +# include: +# - python-version: "3.7.16" +# os: macos-latest +# - python-version: "3.10" +# os: macos-latest +# runs-on: ${{ matrix.os }} +# name: 'prophet (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-prophet/.github/actions/e2e@main +# with: +# working_directory: neptune-prophet +# +# prophet-notify: +# needs: [prophet] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.prophet.result }} +# +# keras: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.10" ] +# os: [ ubuntu-latest, windows-latest ] +# include: +# - python-version: "3.7.16" +# os: macos-latest +# - python-version: "3.10" +# os: macos-latest +# runs-on: ${{ matrix.os }} +# name: 'keras (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-tensorflow-keras/.github/actions/e2e@master +# with: +# working_directory: neptune-tensorflow-keras +# +# keras-notify: +# needs: [keras] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.keras.result }} +# +# lightning: +# env: +# BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} +# USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} +# ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} +# ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} +# SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.10" ] +# os: [ ubuntu-latest ] +# include: +# - python-version: "3.7.16" +# os: macos-latest +# - python-version: "3.10" +# os: macos-latest +# runs-on: ${{ matrix.os }} +# name: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Test +# uses: ./.github/actions/test-e2e +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# module: lightning +# report_job: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' +# +# lightning-notify: +# needs: [lightning] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.lightning.result }} +# +# huggingface: +# env: +# BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} +# USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} +# ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} +# ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} +# SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.10" ] +# os: [ ubuntu-latest ] +# include: +# - python-version: "3.7.16" +# os: macos-latest +# - python-version: "3.10" +# os: macos-latest +# runs-on: ${{ matrix.os }} +# name: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Test +# uses: ./.github/actions/test-e2e +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# module: huggingface +# report_job: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' +# +# huggingface-notify: +# needs: [huggingface] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.huggingface.result }} +# +# zenml: +# env: +# BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} +# USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} +# ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} +# ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} +# SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} +# ZENML_ANALYTICS_OPT_IN: false +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.8" ] +# os: [ ubuntu-latest, macos-latest ] +# runs-on: ${{ matrix.os }} +# name: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Test +# uses: ./.github/actions/test-e2e +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# module: zenml +# report_job: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' +# +# zenml-notify: +# needs: [zenml] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.zenml.result }} +# +# detectron2: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.8", "3.9" ] +# os: [ ubuntu-latest, macos-latest ] +# runs-on: ${{ matrix.os }} +# name: 'detectron2 (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-detectron2/.github/actions/e2e@main +# with: +# working_directory: neptune-detectron2 +# +# detectron2-notify: +# needs: [ detectron2 ] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.detectron2.result }} +# +# rlang: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# os: [ ubuntu-latest, macos-latest ] +# runs-on: ${{ matrix.os }} +# name: 'R (${{ matrix.os }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# os: ${{ matrix.os }} +# python-version: "3.10" +# +# - name: Test integration +# uses: neptune-ai/neptune-r/.github/actions/e2e@master +# with: +# working_directory: rlang +# +# rlang-notify: +# needs: [ rlang ] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.rlang.result }} +# +# sklearn: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.9" ] +# os: [ ubuntu-latest, macos-latest ] +# runs-on: ${{ matrix.os }} +# name: 'sklearn (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-sklearn/.github/actions/e2e@master +# with: +# working_directory: neptune-sklearn +# +# sklearn-notify: +# needs: [ sklearn ] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.sklearn.result }} +# +# xgboost: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.9" ] +# os: [ ubuntu-latest, macos-latest, windows-latest ] +# runs-on: ${{ matrix.os }} +# name: 'xgboost (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-xgboost/.github/actions/e2e@master +# with: +# working_directory: neptune-xgboost +# +# xgboost-notify: +# needs: [ xgboost ] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.xgboost.result }} +# +# sacred: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.7", "3.10" ] +# os: [ ubuntu-latest, macos-latest, windows-latest ] +# runs-on: ${{ matrix.os }} +# name: 'sacred (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-sacred/.github/actions/e2e@main +# with: +# working_directory: neptune-sacred +# +# sacred-notify: +# needs: [ sacred ] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.sacred.result }} +# +# mlflow: +# env: +# NEPTUNE_PROJECT: e2e-tests/integrations +# timeout-minutes: 75 +# strategy: +# fail-fast: false +# matrix: +# python-version: [ "3.8", "3.10" ] +# os: [ ubuntu-latest, macos-latest, windows-latest ] +# runs-on: ${{ matrix.os }} +# name: 'mlflow (${{ matrix.os }} - py${{ matrix.python-version }})' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Install package +# uses: ./.github/actions/install-package +# with: +# python-version: ${{ matrix.python-version }} +# os: ${{ matrix.os }} +# +# - name: Test integration +# uses: neptune-ai/neptune-mlflow/.github/actions/e2e@master +# with: +# working_directory: neptune-mlflow +# +# mlflow-notify: +# needs: [ mlflow ] +# runs-on: ubuntu-latest +# if: (success() || failure()) && github.ref == 'refs/heads/master' +# steps: +# - name: Checkout repository +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# ref: ${{ github.event.client_payload.pull_request.head.ref }} +# +# - name: Notify +# uses: ./.github/actions/job-notify +# with: +# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} +# job-status: ${{ needs.mlflow.result }} + + airflow: env: NEPTUNE_PROJECT: e2e-tests/integrations timeout-minutes: 75 @@ -602,7 +649,7 @@ jobs: python-version: [ "3.8", "3.10" ] os: [ ubuntu-latest, macos-latest, windows-latest ] runs-on: ${{ matrix.os }} - name: 'mlflow (${{ matrix.os }} - py${{ matrix.python-version }})' + name: 'airflow (${{ matrix.os }} - py${{ matrix.python-version }})' steps: - name: Checkout repository uses: actions/checkout@v4 @@ -617,12 +664,12 @@ jobs: os: ${{ matrix.os }} - name: Test integration - uses: neptune-ai/neptune-mlflow/.github/actions/e2e@master + uses: neptune-ai/neptune-airflow/.github/actions/e2e@master with: - working_directory: neptune-mlflow + working_directory: neptune-airflow - mlflow-notify: - needs: [ mlflow ] + airflow-notify: + needs: [ airflow ] runs-on: ubuntu-latest if: (success() || failure()) && github.ref == 'refs/heads/master' steps: @@ -636,4 +683,4 @@ jobs: uses: ./.github/actions/job-notify with: slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} - job-status: ${{ needs.mlflow.result }} + job-status: ${{ needs.airflow.result }} From fdfa1604ca314358b5b1fc149df8c3b3b2ee3a45 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 11:57:13 +0200 Subject: [PATCH 02/14] master -> main --- .github/workflows/integrations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index d8dd679f8..0eca26b13 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -664,7 +664,7 @@ jobs: os: ${{ matrix.os }} - name: Test integration - uses: neptune-ai/neptune-airflow/.github/actions/e2e@master + uses: neptune-ai/neptune-airflow/.github/actions/e2e@main with: working_directory: neptune-airflow From 8b67f4fd4d513cae32615eb53d4617c643db52a3 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 12:12:39 +0200 Subject: [PATCH 03/14] add neptune-airflow to pyproject --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4eef184b4..934bdc210 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ transformers = { version = "*", optional = true } zenml = { version = "*", optional = true, python = "<3.9" } neptune-aws = { version = "*", optional = true } mlflow = { version = ">1.30", optional = true } +neptune-airflow = { version = "*", optional = true } [tool.poetry.extras] aws = ["neptune-aws"] @@ -104,6 +105,7 @@ transformers = ["transformers"] xgboost = ["neptune-xgboost"] zenml = ["zenml"] mlflow = ["mlflow"] +airflow = ["neptune-airflow"] dev = [ "altair", "bokeh", From 7e9e4c428255595166f01da1da0609b6fac75760 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 12:16:18 +0200 Subject: [PATCH 04/14] add airflow installation step --- .github/workflows/integrations.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 0eca26b13..e64b672c7 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -663,6 +663,12 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} + - name: Install airflow + uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + - name: Test integration uses: neptune-ai/neptune-airflow/.github/actions/e2e@main with: From c578cb56c178d61ff3005661bbde8d28bfa6d20d Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 12:28:58 +0200 Subject: [PATCH 05/14] only python-version passed to install airflow --- .github/workflows/integrations.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index e64b672c7..360ad12c9 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -661,7 +661,6 @@ jobs: uses: ./.github/actions/install-package with: python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - name: Install airflow uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main From 733bd4fae2fdff2111f935e038f872b5ac3c6660 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 12:29:08 +0200 Subject: [PATCH 06/14] only python-version passed to install airflow --- .github/workflows/integrations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 360ad12c9..2ef0ce85d 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -661,12 +661,12 @@ jobs: uses: ./.github/actions/install-package with: python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} - name: Install airflow uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main with: python-version: ${{ matrix.python-version }} - os: ${{ matrix.os }} - name: Test integration uses: neptune-ai/neptune-airflow/.github/actions/e2e@main From 1ca027eea70b0ed96f7b2eba2d0df66e3edf20ec Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 12:55:47 +0200 Subject: [PATCH 07/14] temp comment --- .github/workflows/integrations.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 2ef0ce85d..86cdbd218 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -663,15 +663,15 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} - - name: Install airflow - uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main - with: - python-version: ${{ matrix.python-version }} - - - name: Test integration - uses: neptune-ai/neptune-airflow/.github/actions/e2e@main - with: - working_directory: neptune-airflow +# - name: Install airflow +# uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main +# with: +# python-version: ${{ matrix.python-version }} +# +# - name: Test integration +# uses: neptune-ai/neptune-airflow/.github/actions/e2e@main +# with: +# working_directory: neptune-airflow airflow-notify: needs: [ airflow ] From 9ecfe043de0dcd3e63262d90d440e0e5e36cb91e Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 13:00:54 +0200 Subject: [PATCH 08/14] temp comment --- .github/workflows/integrations.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 86cdbd218..034b4d359 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -663,10 +663,10 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} -# - name: Install airflow -# uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main -# with: -# python-version: ${{ matrix.python-version }} + - name: Install airflow + uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main + with: + python-version: ${{ matrix.python-version }} # # - name: Test integration # uses: neptune-ai/neptune-airflow/.github/actions/e2e@main From 7ef5bc41948d9fcf27b14d86779a886f27327c6a Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 13:06:55 +0200 Subject: [PATCH 09/14] checkout --- .github/workflows/integrations.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 034b4d359..6dd0d2e8c 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -663,6 +663,12 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + - name: Install airflow uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main with: From 7628c4f02f5bb99aff5256dde57ed6c7a7e577c7 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 14:49:42 +0200 Subject: [PATCH 10/14] uncomment --- .github/workflows/integrations.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 6dd0d2e8c..2ef0ce85d 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -663,21 +663,15 @@ jobs: python-version: ${{ matrix.python-version }} os: ${{ matrix.os }} - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.client_payload.pull_request.head.ref }} - - name: Install airflow uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main with: python-version: ${{ matrix.python-version }} -# -# - name: Test integration -# uses: neptune-ai/neptune-airflow/.github/actions/e2e@main -# with: -# working_directory: neptune-airflow + + - name: Test integration + uses: neptune-ai/neptune-airflow/.github/actions/e2e@main + with: + working_directory: neptune-airflow airflow-notify: needs: [ airflow ] From 8a34cb8678d824322e7ae4468c7da0ff41b81104 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 15:05:40 +0200 Subject: [PATCH 11/14] working dir --- .github/workflows/integrations.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 2ef0ce85d..0be3c2b71 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -667,6 +667,7 @@ jobs: uses: neptune-ai/neptune-airflow/.github/actions/airflow-install@main with: python-version: ${{ matrix.python-version }} + working_directory: neptune-airflow - name: Test integration uses: neptune-ai/neptune-airflow/.github/actions/e2e@main From ce755de7cffe3daea532530ad6a34bf8da48b874 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 15:24:53 +0200 Subject: [PATCH 12/14] uncomment --- .github/workflows/integrations.yml | 1251 ++++++++++++++-------------- 1 file changed, 625 insertions(+), 626 deletions(-) diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 0be3c2b71..8f50e3ff1 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -5,7 +5,6 @@ on: workflow_dispatch: schedule: - cron: "0 6 * * *" # Run every day at arbitrary time (6:00 AM UTC) - push: env: WORKSPACE_NAME: e2e-tests @@ -13,631 +12,631 @@ env: SLACK_WEBHOOK_URL: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} jobs: -# fastai: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.10" ] -# os: [ ubuntu-latest, windows-latest ] -# include: -# - python-version: "3.7.16" -# os: macos-latest -# - python-version: "3.10" -# os: macos-latest -# runs-on: ${{ matrix.os }} -# name: 'fastai (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-fastai/.github/actions/e2e@master -# with: -# working_directory: neptune-fastai -# -# fastai-notify: -# needs: [fastai] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.fastai.result }} -# -# kedro: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.8", "3.10" ] -# os: [ ubuntu-latest, macos-latest ] -# runs-on: ${{ matrix.os }} -# name: 'kedro (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/kedro-neptune/.github/actions/e2e@main -# with: -# working_directory: kedro-neptune -# -# kedro-notify: -# needs: [kedro] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.kedro.result }} -# -# prophet: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.10" ] -# os: [ ubuntu-latest ] -# include: -# - python-version: "3.7.16" -# os: macos-latest -# - python-version: "3.10" -# os: macos-latest -# runs-on: ${{ matrix.os }} -# name: 'prophet (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-prophet/.github/actions/e2e@main -# with: -# working_directory: neptune-prophet -# -# prophet-notify: -# needs: [prophet] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.prophet.result }} -# -# keras: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.10" ] -# os: [ ubuntu-latest, windows-latest ] -# include: -# - python-version: "3.7.16" -# os: macos-latest -# - python-version: "3.10" -# os: macos-latest -# runs-on: ${{ matrix.os }} -# name: 'keras (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-tensorflow-keras/.github/actions/e2e@master -# with: -# working_directory: neptune-tensorflow-keras -# -# keras-notify: -# needs: [keras] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.keras.result }} -# -# lightning: -# env: -# BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} -# USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} -# ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} -# ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} -# SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.10" ] -# os: [ ubuntu-latest ] -# include: -# - python-version: "3.7.16" -# os: macos-latest -# - python-version: "3.10" -# os: macos-latest -# runs-on: ${{ matrix.os }} -# name: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Test -# uses: ./.github/actions/test-e2e -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# module: lightning -# report_job: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' -# -# lightning-notify: -# needs: [lightning] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.lightning.result }} -# -# huggingface: -# env: -# BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} -# USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} -# ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} -# ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} -# SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.10" ] -# os: [ ubuntu-latest ] -# include: -# - python-version: "3.7.16" -# os: macos-latest -# - python-version: "3.10" -# os: macos-latest -# runs-on: ${{ matrix.os }} -# name: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Test -# uses: ./.github/actions/test-e2e -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# module: huggingface -# report_job: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' -# -# huggingface-notify: -# needs: [huggingface] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.huggingface.result }} -# -# zenml: -# env: -# BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} -# USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} -# ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} -# ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} -# SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} -# ZENML_ANALYTICS_OPT_IN: false -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.8" ] -# os: [ ubuntu-latest, macos-latest ] -# runs-on: ${{ matrix.os }} -# name: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Test -# uses: ./.github/actions/test-e2e -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# module: zenml -# report_job: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' -# -# zenml-notify: -# needs: [zenml] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.zenml.result }} -# -# detectron2: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.8", "3.9" ] -# os: [ ubuntu-latest, macos-latest ] -# runs-on: ${{ matrix.os }} -# name: 'detectron2 (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-detectron2/.github/actions/e2e@main -# with: -# working_directory: neptune-detectron2 -# -# detectron2-notify: -# needs: [ detectron2 ] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.detectron2.result }} -# -# rlang: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# os: [ ubuntu-latest, macos-latest ] -# runs-on: ${{ matrix.os }} -# name: 'R (${{ matrix.os }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# os: ${{ matrix.os }} -# python-version: "3.10" -# -# - name: Test integration -# uses: neptune-ai/neptune-r/.github/actions/e2e@master -# with: -# working_directory: rlang -# -# rlang-notify: -# needs: [ rlang ] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.rlang.result }} -# -# sklearn: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.9" ] -# os: [ ubuntu-latest, macos-latest ] -# runs-on: ${{ matrix.os }} -# name: 'sklearn (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-sklearn/.github/actions/e2e@master -# with: -# working_directory: neptune-sklearn -# -# sklearn-notify: -# needs: [ sklearn ] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.sklearn.result }} -# -# xgboost: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.9" ] -# os: [ ubuntu-latest, macos-latest, windows-latest ] -# runs-on: ${{ matrix.os }} -# name: 'xgboost (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-xgboost/.github/actions/e2e@master -# with: -# working_directory: neptune-xgboost -# -# xgboost-notify: -# needs: [ xgboost ] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.xgboost.result }} -# -# sacred: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.7", "3.10" ] -# os: [ ubuntu-latest, macos-latest, windows-latest ] -# runs-on: ${{ matrix.os }} -# name: 'sacred (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-sacred/.github/actions/e2e@main -# with: -# working_directory: neptune-sacred -# -# sacred-notify: -# needs: [ sacred ] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.sacred.result }} -# -# mlflow: -# env: -# NEPTUNE_PROJECT: e2e-tests/integrations -# timeout-minutes: 75 -# strategy: -# fail-fast: false -# matrix: -# python-version: [ "3.8", "3.10" ] -# os: [ ubuntu-latest, macos-latest, windows-latest ] -# runs-on: ${{ matrix.os }} -# name: 'mlflow (${{ matrix.os }} - py${{ matrix.python-version }})' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Install package -# uses: ./.github/actions/install-package -# with: -# python-version: ${{ matrix.python-version }} -# os: ${{ matrix.os }} -# -# - name: Test integration -# uses: neptune-ai/neptune-mlflow/.github/actions/e2e@master -# with: -# working_directory: neptune-mlflow -# -# mlflow-notify: -# needs: [ mlflow ] -# runs-on: ubuntu-latest -# if: (success() || failure()) && github.ref == 'refs/heads/master' -# steps: -# - name: Checkout repository -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# ref: ${{ github.event.client_payload.pull_request.head.ref }} -# -# - name: Notify -# uses: ./.github/actions/job-notify -# with: -# slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} -# job-status: ${{ needs.mlflow.result }} + fastai: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.10" ] + os: [ ubuntu-latest, windows-latest ] + include: + - python-version: "3.7.16" + os: macos-latest + - python-version: "3.10" + os: macos-latest + runs-on: ${{ matrix.os }} + name: 'fastai (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-fastai/.github/actions/e2e@master + with: + working_directory: neptune-fastai + + fastai-notify: + needs: [fastai] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.fastai.result }} + + kedro: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.8", "3.10" ] + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + name: 'kedro (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/kedro-neptune/.github/actions/e2e@main + with: + working_directory: kedro-neptune + + kedro-notify: + needs: [kedro] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.kedro.result }} + + prophet: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.10" ] + os: [ ubuntu-latest ] + include: + - python-version: "3.7.16" + os: macos-latest + - python-version: "3.10" + os: macos-latest + runs-on: ${{ matrix.os }} + name: 'prophet (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-prophet/.github/actions/e2e@main + with: + working_directory: neptune-prophet + + prophet-notify: + needs: [prophet] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.prophet.result }} + + keras: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.10" ] + os: [ ubuntu-latest, windows-latest ] + include: + - python-version: "3.7.16" + os: macos-latest + - python-version: "3.10" + os: macos-latest + runs-on: ${{ matrix.os }} + name: 'keras (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-tensorflow-keras/.github/actions/e2e@master + with: + working_directory: neptune-tensorflow-keras + + keras-notify: + needs: [keras] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.keras.result }} + + lightning: + env: + BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} + USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} + ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} + ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} + SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.10" ] + os: [ ubuntu-latest ] + include: + - python-version: "3.7.16" + os: macos-latest + - python-version: "3.10" + os: macos-latest + runs-on: ${{ matrix.os }} + name: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Test + uses: ./.github/actions/test-e2e + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + module: lightning + report_job: 'lightning (${{ matrix.os }} - py${{ matrix.python-version }})' + + lightning-notify: + needs: [lightning] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.lightning.result }} + + huggingface: + env: + BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} + USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} + ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} + ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} + SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.10" ] + os: [ ubuntu-latest ] + include: + - python-version: "3.7.16" + os: macos-latest + - python-version: "3.10" + os: macos-latest + runs-on: ${{ matrix.os }} + name: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Test + uses: ./.github/actions/test-e2e + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + module: huggingface + report_job: 'huggingface (${{ matrix.os }} - py${{ matrix.python-version }})' + + huggingface-notify: + needs: [huggingface] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.huggingface.result }} + + zenml: + env: + BUCKET_NAME: ${{ secrets.E2E_BUCKET_NAME }} + USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }} + ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }} + ADMIN_NEPTUNE_API_TOKEN: ${{ secrets.E2E_ADMIN_NEPTUNE_API_TOKEN }} + SERVICE_ACCOUNT_NAME: ${{ secrets.E2E_SERVICE_ACCOUNT_NAME }} + ZENML_ANALYTICS_OPT_IN: false + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.8" ] + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + name: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Test + uses: ./.github/actions/test-e2e + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + module: zenml + report_job: 'zenml (${{ matrix.os }} - py${{ matrix.python-version }})' + + zenml-notify: + needs: [zenml] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.zenml.result }} + + detectron2: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.8", "3.9" ] + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + name: 'detectron2 (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-detectron2/.github/actions/e2e@main + with: + working_directory: neptune-detectron2 + + detectron2-notify: + needs: [ detectron2 ] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.detectron2.result }} + + rlang: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + name: 'R (${{ matrix.os }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + os: ${{ matrix.os }} + python-version: "3.10" + + - name: Test integration + uses: neptune-ai/neptune-r/.github/actions/e2e@master + with: + working_directory: rlang + + rlang-notify: + needs: [ rlang ] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.rlang.result }} + + sklearn: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.9" ] + os: [ ubuntu-latest, macos-latest ] + runs-on: ${{ matrix.os }} + name: 'sklearn (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-sklearn/.github/actions/e2e@master + with: + working_directory: neptune-sklearn + + sklearn-notify: + needs: [ sklearn ] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.sklearn.result }} + + xgboost: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.9" ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + runs-on: ${{ matrix.os }} + name: 'xgboost (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-xgboost/.github/actions/e2e@master + with: + working_directory: neptune-xgboost + + xgboost-notify: + needs: [ xgboost ] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.xgboost.result }} + + sacred: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.7", "3.10" ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + runs-on: ${{ matrix.os }} + name: 'sacred (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-sacred/.github/actions/e2e@main + with: + working_directory: neptune-sacred + + sacred-notify: + needs: [ sacred ] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.sacred.result }} + + mlflow: + env: + NEPTUNE_PROJECT: e2e-tests/integrations + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + python-version: [ "3.8", "3.10" ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + runs-on: ${{ matrix.os }} + name: 'mlflow (${{ matrix.os }} - py${{ matrix.python-version }})' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Install package + uses: ./.github/actions/install-package + with: + python-version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + + - name: Test integration + uses: neptune-ai/neptune-mlflow/.github/actions/e2e@master + with: + working_directory: neptune-mlflow + + mlflow-notify: + needs: [ mlflow ] + runs-on: ubuntu-latest + if: (success() || failure()) && github.ref == 'refs/heads/master' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.client_payload.pull_request.head.ref }} + + - name: Notify + uses: ./.github/actions/job-notify + with: + slack-webhook: ${{ secrets.E2E_INTEGRATIONS_SLACK_WEBHOOK }} + job-status: ${{ needs.mlflow.result }} airflow: env: From 792177be3ec7eb57c255c4ba4e03a38996a6a165 Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Tue, 12 Sep 2023 15:25:53 +0200 Subject: [PATCH 13/14] btw - mlflow fix in pyproject --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 934bdc210..e95c55b8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,7 @@ neptune-xgboost = { version = "*", optional = true } transformers = { version = "*", optional = true } zenml = { version = "*", optional = true, python = "<3.9" } neptune-aws = { version = "*", optional = true } -mlflow = { version = ">1.30", optional = true } +neptune-mlflow = { version = "*", optional = true } neptune-airflow = { version = "*", optional = true } [tool.poetry.extras] @@ -104,7 +104,7 @@ tensorboard = ["neptune-tensorboard"] transformers = ["transformers"] xgboost = ["neptune-xgboost"] zenml = ["zenml"] -mlflow = ["mlflow"] +mlflow = ["neptune-mlflow"] airflow = ["neptune-airflow"] dev = [ "altair", From d50e6005734e972455389d4ac24def811cb6e9bb Mon Sep 17 00:00:00 2001 From: AleksanderWWW Date: Wed, 13 Sep 2023 09:27:05 +0200 Subject: [PATCH 14/14] changelog --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c1604a1e..2811a9429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ -## [UNRELEASED] neptune 1.6.4 +## [UNRELEASED] neptune 1.7.0 + +### Features +- Added support for `airflow` integration ([#1466](https://github.com/neptune-ai/neptune-client/pull/1466)) ### Changes - Add handling of project limits [#1456](https://github.com/neptune-ai/neptune-client/pull/1456)