diff --git a/.evergreen.yml b/.evergreen.yml
index 89bc04e98b..c73e3e3c6c 100644
--- a/.evergreen.yml
+++ b/.evergreen.yml
@@ -318,7 +318,7 @@ functions:
           . preload.sh
           curl -sSfL "$ARTIFACT_URL" > mongosh.zip
           unzip mongosh.zip
-          ./bin/mongosh --smokeTests
+          ./mongosh-*/bin/mongosh --smokeTests
           }
   test_artifact_rpmextract:
     - command: shell.exec
diff --git a/.evergreen/test-package-win32.sh b/.evergreen/test-package-win32.sh
index 2250a71c0a..56d7bc3a11 100644
--- a/.evergreen/test-package-win32.sh
+++ b/.evergreen/test-package-win32.sh
@@ -2,7 +2,7 @@
 if echo "$ARTIFACT_URL" | grep -q .zip; then
   curl -sSfL "$ARTIFACT_URL" > mongosh.zip
   export ARTIFACT_PATH="$PWD/mongosh.zip"
-  (cd /cygdrive/c/Program\ Files/ && rm -rf mongosh && mkdir mongosh && cd mongosh && unzip "$ARTIFACT_PATH" && chmod -v +x bin/*)
+  (cd /cygdrive/c/Program\ Files/ && rm -rf mongosh && mkdir mongosh && cd mongosh && unzip "$ARTIFACT_PATH" && mv -v */bin . && chmod -v +x bin/*)
   export PATH="/cygdrive/c/Program Files/mongosh/bin:$PATH"
 else
   curl -sSfL "$ARTIFACT_URL" > mongosh.msi
diff --git a/packages/build/src/packaging/package/tarball.spec.ts b/packages/build/src/packaging/package/tarball.spec.ts
index 87e6ce3bdf..7ba492a271 100644
--- a/packages/build/src/packaging/package/tarball.spec.ts
+++ b/packages/build/src/packaging/package/tarball.spec.ts
@@ -14,7 +14,7 @@ describe('package tarball', () => {
     const tarname = path.basename(tarball.path).replace(/\.tgz$/, '');
 
     const unzip = spawnSync('tar', [
-      'tf', tarball.path
+      'tf', tarball.path, ...(process.platform === 'win32' ? ['--force-local'] : [])
     ], { encoding: 'utf-8' });
     expect(unzip.error).to.be.undefined;
     expect(unzip.stderr).to.be.empty;