PR_validation.yml

name: PR Validation

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

on:

  pull_request:
    branches: [ main ]
    paths:
      - 'scala/serving/pom.xml'
      - 'scala/ppml/pom.xml'
      - 'scala/orca/pom.xml'
      - 'scala/friesian/pom.xml'
      - 'scala/dllib/pom.xml'
      - 'scala/pom.xml'
      - 'scala/friesian/**'
      - '.github/actions/friesian-scala-ut-action/action.yml'
      - 'scala/dllib/**'
      - 'python/dllib/**'
      - '.github/actions/dllib-scala-ut-action/action.yml'
      - 'scala/ppml/**'
      - '.github/actions/ppml-scala-ut-action/action.yml'
      - 'python/orca/**'
      - '.github/actions/orca/**'
      - 'python/ppml/**'
      - '.github/actions/ppml/**'
      - 'python/friesian/**'
      - '.github/actions/friesian/**'
      - '.github/workflows/PR_validation.yml'
      
  push:
    branches: [ main ]
    paths:
      - 'scala/serving/pom.xml'
      - 'scala/ppml/pom.xml'
      - 'scala/orca/pom.xml'
      - 'scala/friesian/pom.xml'
      - 'scala/dllib/pom.xml'
      - 'scala/pom.xml'
      - 'scala/friesian/**'
      - '.github/actions/friesian-scala-ut-action/action.yml'
      - 'scala/dllib/**'
      - 'python/dllib/**'
      - '.github/actions/dllib-scala-ut-action/action.yml'
      - 'scala/ppml/**'
      - '.github/actions/ppml-scala-ut-action/action.yml'
      - 'python/orca/**'
      - '.github/actions/orca/**'
      - 'python/ppml/**'
      - '.github/actions/ppml/**'
      - 'python/friesian/**'
      - '.github/actions/friesian/**'
      - '.github/workflows/PR_validation.yml'

permissions:
  contents: read
  packages: write

jobs:
  changes:
    name: Paths filter
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]
    outputs:
      friesian-pytest: ${{ steps.filter.outputs.friesian-pytest }}
      friesian-example: ${{ steps.filter.outputs.friesian-example }}
      dllib: ${{ steps.filter.outputs.dllib }}
      orca-pytest: ${{ steps.filter.outputs.orca-pytest }}
      orca-pytest-openvino: ${{ steps.filter.outputs.orca-pytest-openvino }}
      orca-tutorial: ${{ steps.filter.outputs.orca-tutorial }}
      orca-example: ${{ steps.filter.outputs.orca-example }}
      ppml: ${{ steps.filter.outputs.ppml }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            friesian-pytest:
              - 'scala/orca/pom.xml'
              - 'scala/friesian/pom.xml'
              - 'scala/dllib/pom.xml'
              - 'scala/pom.xml'
              - 'scala/friesian/**'
              - '.github/actions/friesian-scala-ut-action/action.yml'
              - 'python/friesian/src/**'
              - 'python/friesian/dev/**'
              - 'python/friesian/test/**'
              - '.github/actions/friesian/**'

            friesian-example: 
              - 'scala/orca/pom.xml'
              - 'scala/friesian/pom.xml'
              - 'scala/dllib/pom.xml'
              - 'python/friesian/example/**'
              - 'python/friesian/src/**'
              - 'python/friesian/dev/**'
              - '.github/actions/friesian/**'
              - 'python/orca/src/bigdl/orca/data/**'

            dllib:
              - 'scala/serving/pom.xml'
              - 'scala/ppml/pom.xml'
              - 'scala/orca/pom.xml'
              - 'scala/friesian/pom.xml'
              - 'scala/dllib/pom.xml'
              - 'scala/pom.xml'
              - 'scala/dllib/**'
              - 'python/dllib/**'
              - '.github/actions/dllib-scala-ut-action/action.yml'
              
            ppml:
              - 'scala/serving/pom.xml'
              - 'scala/ppml/pom.xml'
              - 'scala/orca/pom.xml'
              - 'scala/friesian/pom.xml'
              - 'scala/dllib/pom.xml'
              - 'scala/pom.xml'
              - 'scala/ppml/**'
              - '.github/actions/ppml-scala-ut-action/action.yml'
              - 'python/ppml/**'
              - 'python/dllib/**'
              - '.github/actions/ppml/**'
              
            orca-pytest:
              - 'python/orca/src/**'
              - 'python/orca/dev/**'
              - 'python/orca/test/**'
              - 'python/dllib/src/**'
              - 'scala/orca/pom.xml'
              - 'scala/dllib/pom.xml'
              - '.github/actions/orca/**'

            orca-pytest-openvino:
              - 'python/orca/src/bigdl/orca/learn/openvino/**'
              - 'python/dllib/src/**'
              - 'python/orca/dev/test/**'
              - '.github/actions/orca/**'
              
            orca-tutorial:
              - 'python/orca/src/**'
              - 'python/orca/dev/**'
              - 'python/orca/tutorial/**'
              - 'python/dllib/src/**'
              - 'scala/orca/pom.xml'
              - 'scala/dllib/pom.xml'
              - '.github/actions/orca/**'
              
            orca-example:
              - 'python/orca/src/**'
              - 'python/orca/example/**'
              - 'python/orca/dev/**'
              - 'python/dllib/src/**'
              - 'scala/orca/pom.xml'
              - 'scala/dllib/pom.xml'
              - '.github/actions/orca/**'

  Friesian-Scala-UT:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-pytest == 'true' }}
    runs-on: [ self-hosted, ubuntu-20.04-lts, CLX, AVX512, Ettenmoors ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK8
        uses: ./.github/actions/jdk-setup-action
      - name: Set up maven
        uses: ./.github/actions/maven-setup-action
      - name: Run test
        uses: ./.github/actions/friesian-scala-ut-action

  Dllib-Scala-UT:
    needs: changes
    if: ${{ needs.changes.outputs.dllib == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK8
        uses: ./.github/actions/jdk-setup-action
      - name: Set up maven
        uses: ./.github/actions/maven-setup-action
      - name: Run test
        uses: ./.github/actions/dllib-scala-ut-action

  PPML-Scala-UT:
    needs: changes
    if: ${{ needs.changes.outputs.ppml == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK8
        uses: ./.github/actions/jdk-setup-action
      - name: Set up maven
        uses: ./.github/actions/maven-setup-action
      - name: Run test
        uses: ./.github/actions/ppml-scala-ut-action

  Orca-Basic-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]
        
    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK8
        uses: ./.github/actions/jdk-setup-action
      - name: Set up maven
        uses: ./.github/actions/maven-setup-action
      - name: Setup env
        uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
      - name: Run test
        uses: ./.github/actions/orca/orca-basic-py37-spark3-action/pr-test
      - name: Remove Env
        if: ${{ always() }}
        uses: ./.github/actions/remove-env

  Orca-Basic-PyTorch-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-basic-pytorch-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Basic-Tf2-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-basic-tf2-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Pytest-Py37-Spark3-Tf1:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-pytest-py37-spark3-tf1-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env
  
  Orca-Pytest-Ray-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-pytest-ray-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Pytest-AutoML-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action 
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-pytest-automl-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Pytest-Ray-Ctx-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest == 'true' }}
    runs-on: [self-hosted, Gondolin-Ctx]

    steps:
    - uses: actions/checkout@v3
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-ray-ctx-py37
    - name: Run Test
      uses: ./.github/actions/orca/orca-pytest-ray-ctx-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Pytest-Openvino-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-pytest-openvino == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-pytest-openvino-py37-spark3
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Tutorial-Notebook-Xshards-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-tutorial == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK8
        uses: ./.github/actions/jdk-setup-action
      - name: Set up maven
        uses: ./.github/actions/maven-setup-action
      - name: Setup env
        uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
      - name: Run test
        uses: ./.github/actions/orca/orca-tutorial-notebook-xshards-py37-spark3-action/pr-test
      - name: Remove Env
        if: ${{ always() }}
        uses: ./.github/actions/remove-env

  Orca-Tutorial-Notebook-Xshards-Image-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-tutorial == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK8
        uses: ./.github/actions/jdk-setup-action
      - name: Set up maven
        uses: ./.github/actions/maven-setup-action
      - name: Setup env
        uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
      - name: Run test
        uses: ./.github/actions/orca/orca-tutorial-notebook-xshards-image-py37-spark3-action/pr-test
      - name: Remove Env
        if: ${{ always() }}
        uses: ./.github/actions/remove-env

  Orca-Tutorial-NCF-Ray-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-tutorial == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-tutorial-ncf-ray-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Orca-Tutorial-NCF-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.orca-tutorial == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action 
    - name: Setup env
      uses: ./.github/actions/orca/setup-env/setup-orca-python-py37-basic
    - name: Run Test
      uses: ./.github/actions/orca/orca-tutorial-ncf-py37-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  PPML-Python-UT-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.ppml == 'true' }}
    runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/ppml/setup-env/setup-ppml-python-ut-spark3
    - name: Run Test
      uses: ./.github/actions/ppml/ppml-python-ut-spark3-action/pr-test
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env
  
  Friesian-Python-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-pytest == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/friesian/setup-env/setup-friesian-python-py37-spark3
    - name: Run Test
      uses: ./.github/actions/friesian/friesian-python-py37-spark3/
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env
    
  Friesian-Python-Py37-Spark3-Pip:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-pytest == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/friesian/setup-env/setup-friesian-python-py37-spark3
    - name: Run Test
      uses: ./.github/actions/friesian/friesian-python-py37-spark3-pip/
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Friesian-Python-ExampleTest-Feature-Tf1-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-example == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/friesian/setup-env/setup-friesian-python-py37-spark3
    - name: Run Test
      uses: ./.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Friesian-Python-ExampleTest-Feature-Tf1-Pip-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-example == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/friesian/setup-env/setup-friesian-python-py37-spark3
    - name: Run Test
      uses: ./.github/actions/friesian/friesian-python-exampletest-feature-tf1-pip-py37-spark3/
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Friesian-Python-ExampleTest-Tf2-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-example == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/friesian/setup-env/setup-friesian-python-tf2-py37-spark3
    - name: Run Test
      uses: ./.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env

  Friesian-Python-ExampleTest-Tf2-Pip-Py37-Spark3:
    needs: changes
    if: ${{ needs.changes.outputs.friesian-example == 'true' }}
    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]

    steps:
    - uses: actions/checkout@v3
    - name: Set up JDK8
      uses: ./.github/actions/jdk-setup-action
    - name: Set up maven
      uses: ./.github/actions/maven-setup-action
    - name: Setup env
      uses: ./.github/actions/friesian/setup-env/setup-friesian-python-tf2-py37-spark3
    - name: Run Test
      uses: ./.github/actions/friesian/friesian-python-exampletest-tf2-pip-py37-spark3
    - name: Remove Env
      if: ${{ always() }}
      uses: ./.github/actions/remove-env