work on tests on CI (#2082)

* work on tests on CI

* test first

* skip long running unit test, ensure we run net7.0 only on linux

* pass filepath to LogFilePath

* update test-reporter

* install azure tools needed for tests

* Junit reporter needs full paths

* rollback spec change for messaging and ignore one test

* Skip Azure Functions feature tests on github actions

* Also ignore App Service feature tests for now

* new test workflow action

* fix expressions

* give up on run-name for now

* update substitution

* separate build from test

* remove --no-build from build DOH

* Reference Microsoft.NET.Test.Sdk explicitly in StaticImplicitInititialization Tests

* re-enable other jobs and separate build/test

* tests dropped checkout action

* job dependency problem

* no longer wait for 'build' will introduce a quicker lint job later

* remove inline reporter

.github/workflows/bootstrap/action.yml

@@ -7,6 +7,10 @@ inputs:
     description: 'Install rust toolchain ("true" or "false")'
     required: false
     default: "false"
+  azure:
+      description: 'Install azure functions tool chain ("true" or "false"'
+      required: false
+      default: "false"
 
 runs:
   using: "composite"
@@ -43,3 +47,4 @@ runs:
       uses: ./.github/workflows/install-dependencies
       with:
           rust: '${{ inputs.rust }}'
+          azure: '${{ inputs.azure }}'

.github/workflows/install-dependencies/action.yml

@@ -7,17 +7,46 @@ inputs:
     description: 'Install rust toolchain ("true" or "false")'
     required: false
     default: "false"
+  azure:
+      description: 'Install azure functions tool chain ("true" or "false")'
+      required: false
+      default: "false"
 
 runs:
   using: "composite"
   steps:
+    # RUST
     - name: Rustup
       if: "${{ inputs.rust == 'true' }}"
       shell: bash
       run: rustup default 1.67.1
 
-    - name: Cargo make
+    # - name: Cargo make
+    #   if: "${{ inputs.rust == 'true' }}"
+    #   shell: bash
+    #   run: cargo install --force cargo-make
+
+    - name: Install cargo-make using cache
+      if: "${{ inputs.rust == 'true' }}"
+      uses: baptiste0928/cargo-install@v2
+      with:
+        crate: cargo-make
+        version: "^0.36.8"
+
+    - uses: Swatinem/rust-cache@v2
       if: "${{ inputs.rust == 'true' }}"
+      with:
+        cache-targets: "false"
+        cache-all-crates: "true"
+
+
+    # AZURE
+    - name: Azure functions core tools
+      if: "${{ inputs.azure == 'true' }}"
       shell: bash
-      run: cargo install --force cargo-make
-
+      run: |
+        wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb
+        sudo dpkg -i packages-microsoft-prod.deb
+        sudo apt-get update
+        sudo apt-get install azure-functions-core-tools-4
+        

.github/workflows/release-main.yml

@@ -5,7 +5,8 @@ on:
     branches: [ "main" ]
 
 permissions:
-  contents: read
+    contents: write
+    issues: write
 
 jobs:
   release:
@@ -19,7 +20,7 @@ jobs:
         rust: 'true'
 
     - name: Package with canary suffix
-      run: .ci/linux/release.sh 
+      run: ./build.sh pack
 
     - name: Prepare feedz.io
       uses: hashicorp/vault-action@v2.4.2

.github/workflows/release.yml

@@ -5,7 +5,8 @@ on:
     types: [published]
 
 permissions:
-  contents: write
+    contents: write
+    issues: write
 
 jobs:
   release:
@@ -19,7 +20,7 @@ jobs:
           rust: 'true'
 
     - name: Package
-      run: .ci/linux/release.sh 
+      run: ./build.sh pack
 
     - name: Prepare Nuget
       uses: hashicorp/vault-action@v2.4.2

.github/workflows/test-iis.yml

@@ -29,6 +29,8 @@ jobs:
 
   test-iis:
     runs-on: windows-latest
+    # TODO disable while working on tests
+    if: ${{ false }}
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/test-linux.yml

@@ -22,46 +22,49 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  build:
+  pack:
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
     - name: Bootstrap Action Workspace
       uses: ./.github/workflows/bootstrap
       with:
+          azure: 'true'
           rust: 'true'
-
-    - name: Package
-      run: .ci/linux/release.sh 
 
+    - name: Package
+      run: ./build.sh pack
+
     - uses: actions/upload-artifact@v3
       if: github.event_name == 'push' && startswith(github.ref, 'refs/heads')
       with:
           name: snapshoty-linux
           path: build/output/*
           retention-days: 1
-
-
-  test:
+
+  tests:
     runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    - name: Bootstrap Action Workspace
-      uses: ./.github/workflows/bootstrap
 
-    - name: Test & coverage
-      run: .ci/linux/test.sh
-
-    - name: Store test results
-      if: success() || failure()
-      uses: actions/upload-artifact@v3
-      with:
-        name: test-results
-        path: test/**/junit-*.xml
-
-  startup-hook-test:
+    steps:
+      - uses: actions/checkout@v3
+      - name: Bootstrap Action Workspace
+        uses: ./.github/workflows/bootstrap
+        with:
+          azure: 'true'
+
+      - name: Build
+        shell: bash
+        run: dotnet build -c Release --verbosity minimal
+
+      - name: 'Tests: Unit'
+        uses: ./.github/workflows/test
+        with:
+            name: 'unit'
+            filter: 'FullyQualifiedName!~Elastic.Apm.StartupHook.Tests&FullyQualifiedName!~Elastic.Apm.Profiler.Managed.Tests'
+
+
+  startup-hook-tests:
     runs-on: ubuntu-latest
 
     steps:
@@ -72,19 +75,16 @@ jobs:
     - name: Build agent-zip
       run: ./build.sh agent-zip
 
-    - name: Startup Hook Tests
-      run: .ci/linux/test-startuphooks.sh
-
-    - name: Store test results
-      if: success() || failure()
-      uses: actions/upload-artifact@v3
+    - name: 'Tests: StartupHooks'
+      uses: ./.github/workflows/test
       with:
-        name: test-results
-        path: test/**/junit-*.xml
+        name: 'startuphooks'
+        project: 'test/Elastic.Apm.StartupHook.Tests/Elastic.Apm.StartupHook.Tests.csproj'
+
 
-  profiler-test:
+  profiler-tests:
     runs-on: ubuntu-latest
-
+    
     steps:
     - uses: actions/checkout@v3
     - name: Bootstrap Action Workspace
@@ -95,9 +95,12 @@ jobs:
     - name: Build profiler
       run: ./build.sh profiler-zip
 
-    - name: Profiler Tests
-      run: .ci/linux/test-profiler.sh
-
+    - name: 'Tests: Profiler'
+      uses: ./.github/workflows/test
+      with:
+          name: 'profiler'
+          project: 'test/Elastic.Apm.Profiler.Managed.Tests/Elastic.Apm.Profiler.Managed.Tests.csproj'
+
     - name: Create Docker Image
       run: .ci/linux/build_docker.sh
 
@@ -106,4 +109,4 @@ jobs:
       uses: actions/upload-artifact@v3
       with:
         name: test-results
-        path: test/**/junit-*.xml
+        path: test/**/junit-*.xml

.github/workflows/test-windows-core.yml

@@ -75,6 +75,8 @@ jobs:
 
   startup-hook-test:
     runs-on: windows-2022
+    # TODO disable while working on tests
+    if: ${{ false }}
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/test/action.yml

@@ -0,0 +1,38 @@
+---
+name: Test
+description: Runs part of the test suite
+
+inputs:
+  name:
+    description: 'Name of the test suite we are running.'
+    required: true
+  filter:
+      description: 'test filter to select tests from solution projects'
+      required: false
+  project:
+      description: 'test filter to select tests from solution projects'
+      required: false
+      default: 'ElasticApmAgent.sln'
+  framework:
+      description: 'test filter to select tests from solution projects'
+      required: false
+      default: 'net7.0'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Test ${{ inputs.name }}
+      shell: bash
+      run: |
+          dotnet test -c Release ${{ inputs.project }} --no-build \
+              --verbosity minimal \
+              ${{ inputs.filter != '' && format('--filter "{0}"', inputs.filter) || '' }} \
+              ${{ inputs.framework != '' && format('-f {0}', inputs.framework) || '' }} \
+              --logger:"junit;LogFilePath=$(readlink -f build/output)/junit-{framework}-{assembly}.xml;MethodFormat=Class;FailureBodyFormat=Verbose" \
+
+    - name: Store test results
+      if: success() || failure()
+      uses: actions/upload-artifact@v3
+      with:
+          name: test-results
+          path: build/output/**/junit-*.xml

RELEASING.md

@@ -87,7 +87,7 @@ In this PR we need to update:
 If required then it's possible to run the release script locally, for such, the credentials are needed to push to the NuGet repo.
 
 ```bash
-.ci/linux/release.sh
+./build.sh pack
 .ci/linux/deploy.sh <API_KEY> <SERVER_URL>
 ```
 

sample/AspNetFullFrameworkSampleApp/AspNetFullFrameworkSampleApp.csproj

@@ -385,4 +385,7 @@
   </ProjectExtensions>
   <!-- https://github.com/aspnet/RoslynCodeDomProvider/issues/51 -->
   <Target Name="CheckIfShouldKillVBCSCompiler" />
+
+  <!-- VS Test wants to see this target -->
+  <Target Name="VSTest" Condition="'$(BuildFullFrameworkAsp)' == 'false'"/>
 </Project>