ci: Add coverage tracking

Adds a `make test_cover` target to the Makefile
which builds pulumi-language-yaml with coverage instrumentation
and runs all tests with coverage tracking.

Note that right now, there are no tests that invoke pulumi-language-yaml
so integration test data is empty.

For CI, adds an coverage option that, when enabled,
will run `make test_cover` instead of `make test`,
and upload the results to codecov.
This option is only enabled for tests invoked for PRs
and by the `/run-acceptance-tests` command.

Additionally, similarly to pulumi/pulumi#13334,
this adds a workflow that runs tests on master with coverage tracking
every 12 hours.

Resolves #464

.github/workflows/periodic-cover.yml

@@ -0,0 +1,27 @@
+name: Periodically capture coverage
+on:
+  workflow_dispatch: {}
+  schedule:
+    # Run every 12 hours.
+    - cron: '0 */12 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    uses: ./.github/workflows/stage-test.yml
+    with:
+      live-test: true
+      coverage: true
+    secrets:
+      PULUMI_ACCESS_TOKEN:     ${{ secrets.PULUMI_ACCESS_TOKEN }}
+      AWS_ACCESS_KEY_ID:       ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY:   ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_CI_ROLE_ARN:         ${{ secrets.AWS_CI_ROLE_ARN }}
+      ARM_CLIENT_ID:           ${{ secrets.ARM_CLIENT_ID }}
+      ARM_CLIENT_SECRET:       ${{ secrets.ARM_CLIENT_SECRET }}
+      ARM_SUBSCRIPTION_ID:     ${{ secrets.ARM_SUBSCRIPTION_ID }}
+      ARM_TENANT_ID:           ${{ secrets.ARM_TENANT_ID }}
+      CODECOV_TOKEN:           ${{ secrets.CODECOV_TOKEN }}

.github/workflows/pr-tests-command.yaml

@@ -28,6 +28,7 @@ jobs:
     with:
       live-test: true
       commit-ref: refs/pull/${{ github.event.client_payload.pull_request.number }}/merge
+      coverage: true
     secrets:
       PULUMI_ACCESS_TOKEN:     ${{ secrets.PULUMI_ACCESS_TOKEN }}
       AWS_ACCESS_KEY_ID:       ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -37,3 +38,4 @@ jobs:
       ARM_CLIENT_SECRET:       ${{ secrets.ARM_CLIENT_SECRET }}
       ARM_SUBSCRIPTION_ID:     ${{ secrets.ARM_SUBSCRIPTION_ID }}
       ARM_TENANT_ID:           ${{ secrets.ARM_TENANT_ID }}
+      CODECOV_TOKEN:           ${{ secrets.CODECOV_TOKEN }}

.github/workflows/pr-tests.yaml

@@ -13,6 +13,7 @@ jobs:
     uses: ./.github/workflows/stage-test.yml
     with:
       live-test: true
+      coverage: true
     secrets:
       PULUMI_ACCESS_TOKEN:     ${{ secrets.PULUMI_ACCESS_TOKEN }}
       AWS_ACCESS_KEY_ID:       ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -22,3 +23,4 @@ jobs:
       ARM_CLIENT_SECRET:       ${{ secrets.ARM_CLIENT_SECRET }}
       ARM_SUBSCRIPTION_ID:     ${{ secrets.ARM_SUBSCRIPTION_ID }}
       ARM_TENANT_ID:           ${{ secrets.ARM_TENANT_ID }}
+      CODECOV_TOKEN:           ${{ secrets.CODECOV_TOKEN }}

.github/workflows/stage-test.yml

@@ -30,6 +30,11 @@ on:
         default: ''
         required: false
         type: string
+      coverage:
+        description: "Collects coverage stats"
+        default: false
+        required: false
+        type: boolean
     secrets:
       PULUMI_ACCESS_TOKEN:   { required: false }
       AWS_ACCESS_KEY_ID:     { required: false }
@@ -39,6 +44,7 @@ on:
       ARM_CLIENT_SECRET:     { required: false }
       ARM_SUBSCRIPTION_ID:   { required: false }
       ARM_TENANT_ID:         { required: false }
+      CODECOV_TOKEN:         { required: false }
 
 
 env:
@@ -121,7 +127,22 @@ jobs:
           path: ${{ steps.list-schemas.outputs.schemas }}
 
       - name: Test
+        if: ${{ !inputs.coverage }}
         run: make test
+      - name: Test with Coverage
+        if: ${{ inputs.coverage }}
+        run: make test_cover
+
+      - name: Upload coverage data
+        if: ${{ inputs.coverage }}
+        uses: codecov/codecov-action@v3
+        with:
+          directory: coverage
+          files: "*"
+          fail_ci_if_error: false
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
     strategy:
       fail-fast: false
       matrix:

.gitignore

@@ -15,3 +15,4 @@ command-output/
 obj/
 __pycache__/
 node_modules/
+/coverage

Makefile

@@ -15,6 +15,8 @@ PLUGIN_VERSION_AZURE_NATIVE := 1.99.1
 PLUGIN_VERSION_RANDOM       := 4.12.1
 GO                          := go
 
+BUILD_FLAGS ?=
+
 .phony: .EXPORT_ALL_VARIABLES
 .EXPORT_ALL_VARIABLES:
 
@@ -58,15 +60,32 @@ lint-copyright:
 
 build:: ensure
 	mkdir -p ./bin
-	${GO} build -o ./bin -p ${CONCURRENCY} ./cmd/...
+	${GO} build $(BUILD_FLAGS) -o ./bin -p ${CONCURRENCY} ./cmd/...
 
 # Ensure that in tests, the language server is accessible
 test:: build get_plugins get_schemas
 	PATH="${PWD}/bin:${PATH}" PULUMI_LIVE_TEST="${PULUMI_LIVE_TEST}" \
-	  ${GO} test -v --timeout 30m -count 1 -race -parallel ${CONCURRENCY} ./pkg/...
+	  ${GO} test -v --timeout 30m -count 1 -race -parallel ${CONCURRENCY} ./...
+
+# Runs tests with code coverage tracking.
+# Two output files are generated in the coverage/ directory:
+#
+#  unit.out        unit test coverage
+#  integration.out integration test coverage
+#
+# Integration test coverage is only generated for tests that invoke the
+# pulumi-language-yaml executable.
+.phony: test_cover
+test_cover: get_plugins get_schemas
+	make build BUILD_FLAGS="-cover -coverpkg=github.com/pulumi/pulumi-yaml/..."
+	rm -rf coverage && mkdir -p coverage
+	$(eval COVERDIR := $(shell mktemp -d))
+	PATH="${PWD}/bin:${PATH}" PULUMI_LIVE_TEST="${PULUMI_LIVE_TEST}" GOCOVERDIR=$(COVERDIR) \
+	     ${GO} test -v -coverprofile=coverage/unit.out -coverpkg=./... ./...
+	go tool covdata textfmt -i=$(COVERDIR) -o=coverage/integration.out
 
 test_short::
-	${GO} test --timeout 30m -short -count 1 -parallel ${CONCURRENCY} ./pkg/...
+	${GO} test --timeout 30m -short -count 1 -parallel ${CONCURRENCY} ./...
 
 test_live:: PULUMI_LIVE_TEST = true
 test_live:: test_live_prereq test