[SYCL] CI changes for In-Tree End-to-End tests

I'm extending our test matrix with `in_tree_e2e` parameter and updating
llvm_test_suite action/sycl_linux_build_and_test workflow to respect its
value.

Author: aelovikov-intel

.github/workflows/sycl_linux_build_and_test.yml

@@ -160,6 +160,8 @@ jobs:
       run: tar -cJf llvm_sycl.tar.xz -C $GITHUB_WORKSPACE/build/install .
     - name: Pack LIT
       run: tar -cJf lit.tar.xz -C $GITHUB_WORKSPACE/src/llvm/utils/lit .
+    - name: Pack End-to-End LIT tests
+      run: tar -cJf e2e_tests.tar.xz -C $GITHUB_WORKSPACE/src/sycl/test-e2e .
     - name: Upload toolchain
       uses: actions/upload-artifact@v3
       with:
@@ -170,6 +172,11 @@ jobs:
       with:
         name: sycl_lit_${{ inputs.build_artifact_suffix }}
         path: lit.tar.xz
+    - name: Upload End-to-End LIT tests
+      uses: actions/upload-artifact@v3
+      with:
+        name: sycl_e2e_tests_${{ inputs.build_artifact_suffix }}
+        path: e2e_tests.tar.xz
 
   aws-start:
     name: Start AWS
@@ -241,9 +248,12 @@ jobs:
         sycl_archive: llvm_sycl.tar.xz
         lit_artifact: sycl_lit_${{ inputs.build_artifact_suffix }}
         lit_archive: lit.tar.xz
+        e2e_tests_artifact: sycl_e2e_tests_${{ inputs.build_artifact_suffix }}
+        e2e_tests_archive: e2e_tests.tar.xz
         check_sycl_all: ${{ matrix.check_sycl_all }}
         results_name_suffix: ${{ matrix.config }}_${{ inputs.build_artifact_suffix }}
         cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
+        in_tree_e2e: ${{ matrix.in_tree_e2e }}
 
   khronos_sycl_cts:
     needs: build

devops/actions/llvm_test_suite/action.yml

@@ -19,6 +19,13 @@ inputs:
     description: 'Name of LIT archive file'
     required: false
     default: 'lit.tar.xz'
+  e2e_tests_artifact:
+    description: 'Name of the artifact, that contains End-to-End LIT Tests'
+    required: true
+  e2e_tests_archive:
+    description: 'Name of End-to-End LIT Tests archive file'
+    required: false
+    default: 'e2e_tests.tar.xz'
   results_name_suffix:
     description: 'Name suffix of the results artifact'
     required: true
@@ -28,6 +35,10 @@ inputs:
   cmake_args:
     description: 'Extra arguments to cmake command'
     required: false
+  in_tree_e2e:
+    description: "Flag directing to use in-tree End-to-End LIT tests"
+    required: false
+    default: false
 
 post-if: false
 runs:
@@ -66,13 +77,33 @@ runs:
       ls -la
       tar -xf ${{ inputs.lit_archive }} -C lit
       rm -f ${{ inputs.lit_archive }}
+  - name: Download End-to-End LIT Tests
+    if: inputs.in_tree_e2e
+    uses: actions/download-artifact@v3
+    with:
+      name: ${{ inputs.e2e_tests_artifact }}
+  - name: Extract End-to-End LIT Tests
+    if: inputs.in_tree_e2e
+    shell: bash
+    run: |
+      mkdir test_e2e
+      ls -la
+      tar -xf ${{ inputs.e2e_tests_archive }} -C test_e2e
+      ls -R test_e2e
+      rm -f ${{ inputs.e2e_tests_archive }}
   - name: Configure
     shell: bash
     run: |
       echo "::group::CMake configuration"
       mkdir build
       export PATH=$PWD/toolchain/bin/:$PATH
-      cmake -GNinja -B./build -S./llvm_test_suite -DTEST_SUITE_SUBDIRS=SYCL -DCHECK_SYCL_ALL="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DTEST_SUITE_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      if ${{ inputs.in_tree_e2e }}
+      then
+        # TODO: Rename check_sycl_all input
+        cmake  -GNinja -B./build -S./test_e2e -DSYCL_TEST_E2E_TARGETS="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DLLVM_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      else
+        cmake -GNinja -B./build -S./llvm_test_suite -DTEST_SUITE_SUBDIRS=SYCL -DCHECK_SYCL_ALL="${{ inputs.check_sycl_all }}" -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" -DTEST_SUITE_LIT="$PWD/lit/lit.py" ${{ inputs.cmake_args }}
+      fi
       echo "::endgroup::"
   - name: Run testing
     shell: bash
@@ -81,7 +112,10 @@ runs:
       export PATH=$PWD/toolchain/bin/:$PATH
       # TODO make this part of container build
       export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/hip/lib/:/opt/rocm/lib
-      export LIT_OPTS="-v --no-progress-bar --show-unsupported --max-time 3600 --time-tests -o $PWD/build/results_${{ inputs.results_name_suffix }}.json"
+      # For standalone llvm-test-suite
+      export LIT_OPTS="-v --no-progress-bar --show-unsupported --max-time 3600 --time-tests -o $PWD/build/results_${{ inputs.results_name_suffix }}_${{ inputs.in_tree_e2e }}.json"
+      # For in-tree End-to-End tests
+      export SYCL_TEST_E2E_TARGETS="-v --no-progress-bar --show-unsupported --max-time 3600 --time-tests -o $PWD/build/results_${{ inputs.results_name_suffix }}_${{ inputs.in_tree_e2e }}.json"
       if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
         source /runtimes/oneapi-tbb/env/vars.sh;
       elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
@@ -103,13 +137,18 @@ runs:
       SYCL_PI_TRACE=-1 sycl-ls
       echo "::endgroup::"
       cd build
-      ninja check-sycl-all
+      if ${{ inputs.in_tree_e2e }}
+      then
+        ninja check-sycl-e2e
+      else
+        ninja check-sycl-all
+      fi
   - name: Upload test results
     uses: actions/upload-artifact@v1
     if: always()
     with:
       name: lit_results
-      path: build/results_${{ inputs.results_name_suffix }}.json
+      path: build/results_${{ inputs.results_name_suffix }}_${{ inputs.in_tree_e2e}}.json
   - name: Cleanup
     shell: bash
     if: always()
@@ -118,3 +157,4 @@ runs:
       rm -rf lit
       rm -rf build
       rm -rf llvm_test_suite
+      rm -rf test-e2e

devops/scripts/generate_test_matrix.js

@@ -47,8 +47,16 @@ module.exports = ({core, process}) => {
           } else {
             v["env"] = {};
           }
+          // Temporary support for both in-tree/out-of-tree (llvm-test-suite)
+          // end-to-end tests.
+          v["in_tree_e2e"] = false;
           enabledLTSConfigs.push(v);
           if (v["aws-type"]) enabledLTSAWSConfigs.push(v);
+
+          var inTreeV = Object.assign({}, v);
+          inTreeV["in_tree_e2e"] = true;
+          inTreeV["name"] = inTreeV["name"] + " In-Tree"
+          enabledLTSConfigs.push(inTreeV);
         }
       });