Merge #10815 #10821 #10822 #10823

10815: ci: Pin yarn lockfile for security & dependency scanning r=AaronFriel a=AaronFriel

Pins our yarn.lock for the node SDK. Though we are publishing a library and that does mean the yarn.lock is not used by consumers of our library, we obtain a few benefits:

1. Test consistency, we can be sure that from us merging a PR to shipping an SDK the dependencies used in tests.
2. Dependabot/Snyk/etc. dependency scanners prefer lockfiles, as they can provide the scanner with information about transitive dependencies and alert us to necessary updates. This could lead to us updating or pinning dependencies in package.json.
3. We can use a yarn.lock to produce an SBOM for our SDK, which will help the product satisfy coming requirements under [United States Executive Order 14028, Improving the Nation's Cybersecurity](https://www.nist.gov/itl/executive-order-14028-improving-nations-cybersecurity).

Resolves #10619 

10821: ci: Updated test repo, check if Windows tests pass r=AaronFriel a=AaronFriel

The remote repo has been updated to use the latest Go & Pulumi SDK, so this ought to unblock these tests.

Resolves #8646.

10822: ci: Remove several test skips, check if unnecessary r=AaronFriel a=AaronFriel

These tests have been skipped for some time on various platforms, this should check if we _need_ to skip them or if we can return to testing these on Windows or all platforms.

Resolves #7869.
Resolves #3811.

10823: ci: Re-enable Windows tests with temp dir r=AaronFriel a=AaronFriel

These tests pass on my Windows device, perhaps due to some more systemic fix with path handling.

Resume running these tests.

Resolves #8649

Co-authored-by: Aaron Friel <mayreply@aaronfriel.com>

.github/workflows/ci-build-sdks.yml

@@ -92,7 +92,7 @@ jobs:
         with:
           node-version: ${{ fromJson(inputs.version-set).nodejs }}
           cache: yarn
-          cache-dependency-path: sdk/nodejs/*.json
+          cache-dependency-path: sdk/nodejs/yarn.lock
       - name: Install yarn
         run: |
           npm install -g yarn

.github/workflows/ci-lint.yml

@@ -73,7 +73,7 @@ jobs:
         with:
           node-version: ${{ fromJson(inputs.version-set).nodejs }}
           cache: yarn
-          cache-dependency-path: sdk/nodejs/package.json
+          cache-dependency-path: sdk/nodejs/yarn.lock
       - name: Install Python deps
         run: |
           python -m pip install --upgrade pip requests wheel urllib3 chardet

.github/workflows/ci-run-test.yml

@@ -157,7 +157,14 @@ jobs:
         with:
           node-version: ${{ fromJson(inputs.version-set).nodejs }}
           cache: yarn
-          cache-dependency-path: sdk/nodejs/*.json
+          cache-dependency-path: sdk/nodejs/yarn.lock
+      - name: Uninstall pre-installed Pulumi (windows)
+        if: inputs.platform == 'windows-latest'
+        run: |
+          if command -v pulumi.exe; then
+            echo "Deleting pulumi"
+            rm -rf "$(command -v pulumi.exe)/../pulumi*"
+          fi
       - name: Install yarn
         run: |
           npm install -g yarn

pkg/cmd/pulumi/package.go

@@ -50,8 +50,7 @@ Subcommands of this command are useful to package authors during development.`,
 // schema source is either a file (ending with .[json|y[a]ml]) or a plugin with an
 // optional version:
 //
-//		FILE.[json|y[a]ml] | PLUGIN[@VERSION] | PATH_TO_PLUGIN
-//
+//	FILE.[json|y[a]ml] | PLUGIN[@VERSION] | PATH_TO_PLUGIN
 func schemaFromSchemaSource(packageSource string) (*schema.Package, error) {
 	var spec schema.PackageSpec
 	wd, err := os.Getwd()

scripts/get-job-matrix.py

@@ -150,13 +150,16 @@ def run_list_tests(pkg_dir: str, tags: List[str]) -> List[str]:
     # $ go test -tags all --list ./tests/integration
     # no Go files in /home/friel/c/github.com/pulumi/pulumi
     # ```
-    cmd = sp.run(
-        ["go", "test", "-tags", " ".join(tags), "--list", "."],
-        check=True,
-        cwd=pkg_dir,
-        capture_output=True,
-        text=True,
-    )
+    try:
+        cmd = sp.run(
+            ["go", "test", "-tags", " ".join(tags), "--list", "."],
+            check=True,
+            cwd=pkg_dir,
+            capture_output=True,
+            text=True,
+        )
+    except sp.CalledProcessError as err:
+        raise Exception("Failed to list packages in module, usually this implies a Go compilation error. Check that `make lint` succeeds.") from err
 
     tests: List[str] = []
 

sdk/go/auto/local_workspace_test.go

@@ -53,7 +53,6 @@ var pulumiOrg = getTestOrg()
 const pName = "testproj"
 const agent = "pulumi/pulumi/test"
 const pulumiTestOrg = "pulumi-test"
-const windows = "windows"
 
 func TestWorkspaceSecretsProvider(t *testing.T) {
 	t.Parallel()
@@ -441,10 +440,6 @@ func randomStackName() string {
 func TestNewStackRemoteSource(t *testing.T) {
 	t.Parallel()
 
-	if runtime.GOOS == windows {
-		t.Skip("TODO[pulumi/pulumi#8646] update github.com/pulumi/test-repo to fix Go compilation on Windows")
-	}
-
 	ctx := context.Background()
 	pName := "go_remote_proj"
 	sName := randomStackName()
@@ -540,10 +535,6 @@ func TestNewStackRemoteSource(t *testing.T) {
 func TestUpsertStackRemoteSource(t *testing.T) {
 	t.Parallel()
 
-	if runtime.GOOS == windows {
-		t.Skip("TODO[pulumi/pulumi#8646] update github.com/pulumi/test-repo to fix Go compilation on Windows")
-	}
-
 	ctx := context.Background()
 	pName := "go_remote_proj"
 	sName := randomStackName()
@@ -639,10 +630,6 @@ func TestUpsertStackRemoteSource(t *testing.T) {
 func TestNewStackRemoteSourceWithSetup(t *testing.T) {
 	t.Parallel()
 
-	if runtime.GOOS == windows {
-		t.Skip("TODO[pulumi/pulumi#8646] update github.com/pulumi/test-repo to fix Go compilation on Windows")
-	}
-
 	ctx := context.Background()
 	pName := "go_remote_proj"
 	sName := randomStackName()
@@ -657,6 +644,9 @@ func TestNewStackRemoteSourceWithSetup(t *testing.T) {
 		},
 	}
 	binName := "examplesBinary"
+	if runtime.GOOS == "windows" {
+		binName = binName + ".exe"
+	}
 	repo := GitRepo{
 		URL:         "https://github.com/pulumi/test-repo.git",
 		ProjectPath: "goproj",
@@ -750,10 +740,6 @@ func TestNewStackRemoteSourceWithSetup(t *testing.T) {
 func TestUpsertStackRemoteSourceWithSetup(t *testing.T) {
 	t.Parallel()
 
-	if runtime.GOOS == windows {
-		t.Skip("TODO[pulumi/pulumi#8646] update github.com/pulumi/test-repo to fix Go compilation on Windows")
-	}
-
 	ctx := context.Background()
 	pName := "go_remote_proj"
 	sName := randomStackName()
@@ -768,6 +754,9 @@ func TestUpsertStackRemoteSourceWithSetup(t *testing.T) {
 		},
 	}
 	binName := "examplesBinary"
+	if runtime.GOOS == "windows" {
+		binName = binName + ".exe"
+	}
 	repo := GitRepo{
 		URL:         "https://github.com/pulumi/test-repo.git",
 		ProjectPath: "goproj",

sdk/go/common/workspace/plugins_install_test.go

@@ -25,7 +25,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"runtime"
 	"sync"
 	"testing"
 
@@ -171,10 +170,6 @@ func testPluginInstall(t *testing.T, expectedDir string, files map[string][]byte
 }
 
 func TestInstallNoDeps(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("TODO[pulumi/pulumi#8649] Skipped on Windows: issues with TEMP dir")
-	}
-
 	name := "foo.txt"
 	content := []byte("hello\n")
 
@@ -194,10 +189,6 @@ func TestInstallNoDeps(t *testing.T) {
 }
 
 func TestReinstall(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("TODO[pulumi/pulumi#8649] Skipped on Windows: issues with TEMP dir")
-	}
-
 	name := "foo.txt"
 	content := []byte("hello\n")
 
@@ -228,10 +219,6 @@ func TestReinstall(t *testing.T) {
 }
 
 func TestConcurrentInstalls(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("TODO[pulumi/pulumi#8649] Skipped on Windows: issues with TEMP dir")
-	}
-
 	name := "foo.txt"
 	content := []byte("hello\n")
 

sdk/nodejs/.gitignore

@@ -5,3 +5,4 @@
 /custom_node/
 /runtime/native/node_dev/
 .nyc_output/
+!yarn.lock

sdk/nodejs/Makefile

@@ -24,7 +24,7 @@ GO_TEST_FAST = $(PYTHON) ../../scripts/go-test.py $(GO_TEST_FAST_FLAGS)
 
 ensure:: yarn.ensure node.ensure .ensure.phony
 .ensure.phony: package.json
-	yarn install
+	yarn install --frozen-lockfile
 	@touch .ensure.phony
 
 lint:: ensure