Merge pull request #164 from semgrep/austin/saf-1199-vendor-pro-binar…

…y-with-vscode

chore: vendor pro binary with vscode

.github/workflows/publish.yaml

@@ -1,28 +1,61 @@
 on:
-  push:
-    tags:
-      - "*"
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      preRelease:
+        description: "Is this a pre-release?"
+        type: boolean
+        required: false
+        default: false
+      dryRun:
+        description: "Is this a dry run?"
+        type: boolean
+        required: false
+        default: false
+permissions:
+  id-token: write
+  contents: read
 
-name: Deploy Extension
 jobs:
-  deploy:
+  vscode-extension-publish:
+    strategy:
+      matrix:
+        target: [linux-x64, linux-arm64, darwin-x64, darwin-arm64]
+    name: Deploy Extension
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v4.0.3
         with:
-          node-version: 16
+          node-version: lts/*
       - run: npm ci
+      - uses: "aws-actions/configure-aws-credentials@v4.0.2"
+        with:
+          role-to-assume: "arn:aws:iam::338683922796:role/semgrep-ide-integration-deploy-role"
+          role-duration-seconds: 900,
+          role-session-name: "semgrep-ide-integration-deploy"
+          aws-region: "us-west-2"
+      - name: download osemgrep pro
+        run: ./download-osemgrep-pro.sh ${{ matrix.target }}
       - name: Publish to Open VSX Registry
-        uses: HaaLeo/publish-vscode-extension@v1
+        uses: HaaLeo/publish-vscode-extension@v1.6.2
         id: publishToOpenVSX
         with:
           # Tied to austin@, lives in 1password, does not expire
           pat: ${{ secrets.OPEN_VSX_TOKEN }}
+          target: ${{ matrix.target }} # only for specific platforms
+          # release pre-release if that's the event
+          preRelease: ${{github.event.release.prerelease || inputs.preRelease}}
+          dryRun: ${{ inputs.dryRun }}
       - name: Publish to Visual Studio Marketplace
-        uses: HaaLeo/publish-vscode-extension@v1
+        uses: HaaLeo/publish-vscode-extension@v1.6.2
         with:
           # Tied to bence@, lives in 1password expires in may 2024
           pat: ${{ secrets.VS_MARKETPLACE_TOKEN }}
           registryUrl: https://marketplace.visualstudio.com
           extensionFile: ${{ steps.publishToOpenVSX.outputs.vsixPath }}
+          target: ${{ matrix.target }}
+          # release pre-release if that's the event
+          preRelease: ${{github.event.release.prerelease || inputs.preRelease}}
+          dryRun: ${{ inputs.dryRun }}

.github/workflows/tests.yml

@@ -10,6 +10,9 @@ on:
       - master
     paths-ignore:
       - "**.md"
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   vsce-test:
@@ -25,11 +28,23 @@ jobs:
       - name: Install Semgrep
         run: python -m pip install semgrep
       - name: Setup
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4.0.3
         with:
-          node-version: 16.x
+          node-version: lts/*
       - name: install dependencies
         run: npm ci
+      - uses: "aws-actions/configure-aws-credentials@v4.0.2"
+        with:
+          role-to-assume: "arn:aws:iam::338683922796:role/semgrep-ide-integration-deploy-role"
+          role-duration-seconds: 900,
+          role-session-name: "semgrep-ide-integration-deploy"
+          aws-region: "us-west-2"
+      - if: matrix.os == 'macos-latest'
+        name: download osemgrep pro
+        run: ./download-osemgrep-pro.sh darwin-arm64
+      - if: matrix.os == 'ubuntu-latest'
+        name: download osemgrep pro
+        run: ./download-osemgrep-pro.sh linux-x64
       - name: pretest
         run: npm run pretest
       - name: run native tests (Linux)
@@ -39,7 +54,7 @@ jobs:
         if: matrix.os == 'macos-latest'
         run: npm run test-native
       - name: uninstall semgrep
-        run: python -m pip uninstall -y semgrep
+        run: python -m pip uninstall -y semgrep && rm dist/osemgrep-pro
       - name: run js tests (Linux)
         if: matrix.os == 'ubuntu-latest'
         run: xvfb-run -a npm run test-js
@@ -58,9 +73,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4.0.3
         with:
-          node-version: 16.x
+          node-version: lts/*
       - name: install dependencies
         run: npm ci
         # This is needed so download-lspjs.sh can run
@@ -83,8 +98,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v4.0.3
+        with:
+          node-version: lts/*
+      - uses: "aws-actions/configure-aws-credentials@v4.0.2"
         with:
-          node-version: 16
+          role-to-assume: "arn:aws:iam::338683922796:role/semgrep-ide-integration-deploy-role"
+          role-duration-seconds: 900,
+          role-session-name: "semgrep-ide-integration-deploy"
+          aws-region: "us-west-2"
+      - name: download osemgrep pro
+        run: ./download-osemgrep-pro.sh linux-x64
       - run: npm ci
       - run: npm run vscode:prepublish

.gitignore

@@ -10,7 +10,7 @@ node_modules
 
 # Misc
 .DS_Store
-lspjs
+dist/*
 # From testing
 *_results.json
 

.vscode/extensions.json

@@ -1,3 +1,7 @@
 {
-  "recommendations": ["Semgrep.semgrep", "Tobermory.es6-string-html"]
+"recommendations": [
+    "Semgrep.semgrep",
+    "Tobermory.es6-string-html",
+    "connor4312.esbuild-problem-matchers"
+]
 }

.vscode/launch.json

@@ -8,13 +8,13 @@
       "runtimeExecutable": "${execPath}",
       "args": ["--extensionDevelopmentPath=${workspaceRoot}"],
       "outFiles": ["${workspaceRoot}/out/**/*.js"],
-      "preLaunchTask": "watch-build"
+      "preLaunchTask": "watch"
     },
     {
       "type": "node",
       "request": "attach",
       "name": "Attach to Server",
-      "port": 6009,
+      "port": 9229,
       "restart": true,
       "outFiles": [
         "${workspaceRoot}/lspjs/out/**/*.js",
@@ -34,7 +34,7 @@
       "outFiles": ["${workspaceRoot}/out/**/*.js"],
       "preLaunchTask": {
         "type": "npm",
-        "script": "test-compile"
+        "script": "esbuild"
       }
     }
   ]

.vscode/tasks.json

@@ -2,43 +2,39 @@
   "version": "2.0.0",
   "tasks": [
     {
-      "type": "npm",
-      "script": "compile",
-      "group": "build",
+      "label": "watch",
+      "dependsOn": ["npm: watch:tsc", "npm: watch:esbuild"],
       "presentation": {
-        "panel": "dedicated",
         "reveal": "never"
       },
-      "problemMatcher": ["$tsc"]
+      "group": {
+        "kind": "build",
+        "isDefault": true
+      }
     },
     {
       "type": "npm",
-      "label": "watch-build",
-      "script": "esbuild-watch",
+      "label": "npm: watch:esbuild",
+      "script": "watch:esbuild",
       "isBackground": true,
-      "problemMatcher": {
-        "owner": "typescript",
-        "fileLocation": "relative",
-        "pattern": {
-          "regexp": "^([^\\s].*)\\((\\d+|\\d+,\\d+|\\d+,\\d+,\\d+,\\d+)\\):\\s+(error|warning|info)\\s+(TS\\d+)\\s*:\\s*(.*)$",
-          "file": 1,
-          "location": 2,
-          "severity": 3,
-          "code": 4,
-          "message": 5
-        },
-        "background": {
-          "activeOnStart": true,
-          "beginsPattern": "\\[watch\\] build started.*",
-          "endsPattern": "\\[watch\\] build finished.*"
-        }
+      "problemMatcher": "$esbuild-watch",
+      "presentation": {
+        "reveal": "never"
       },
       "group": {
         "kind": "build",
         "isDefault": true
-      },
+      }
+    },
+    {
+      "type": "npm",
+      "script": "watch:tsc",
+      "group": "build",
+      "problemMatcher": "$tsc-watch",
+      "isBackground": true,
+      "label": "npm: watch:tsc",
       "presentation": {
-        "panel": "dedicated",
+        "group": "watch",
         "reveal": "never"
       }
     }

.vscodeignore

@@ -1,7 +1,16 @@
 .vscode/**
 .vscode-test/**
 out/test/**
+out/semgrep-lsp.js
 src/**
+node_modules/**
+.pre-commit-config.yaml
+build.mjs
+download-lspjs.sh
+download-osemgrep-pro.sh
+eslint.config.js
+images/**
+.github/**
 .gitignore
 vsc-extension-quickstart.md
 **/tsconfig.json

build.mjs

@@ -6,6 +6,8 @@ import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin";
 async function buildSentrySourceMap() {
   esbuild.build({
     sourcemap: true, // Source map generation must be turned on
+    bundle: true,
+    platform: "node",
     plugins: [
       // Put the Sentry esbuild plugin after all other plugins
       sentryEsbuildPlugin({
@@ -16,7 +18,8 @@ async function buildSentrySourceMap() {
     ],
   });
 }
-async function buildExtension(watch) {
+
+async function buildExtension(watch, sourcemap, minify) {
   const options = {
     logLevel: "info",
     entryPoints: ["./src/extension.ts"],
@@ -25,7 +28,9 @@ async function buildExtension(watch) {
     platform: "node",
     format: "cjs",
     external: ["vscode"],
-    sourcemap: isSourcemap,
+    sourcemap,
+    plugins: [esbuildProblemMatcherPlugin],
+    minify,
   };
   if (watch) {
     let ctx = await esbuild.context(options);
@@ -34,14 +39,16 @@ async function buildExtension(watch) {
     await esbuild.build(options);
   }
 }
-async function buildWebview(watch) {
+async function buildWebview(watch, sourcemap, minify) {
   let options = {
     logLevel: "info",
     entryPoints: ["./src/webview-ui/index.tsx"],
     outfile: "./out/webview.js",
     bundle: true,
+    platform: "node",
     plugins: [cssModulesPlugin()],
-    sourcemap: isSourcemap,
+    sourcemap,
+    minify,
   };
   if (watch) {
     let ctx = await esbuild.context(options);
@@ -53,9 +60,30 @@ async function buildWebview(watch) {
 
 const isWatch = process.argv.includes("--watch");
 const isSourcemap = process.argv.includes("--sourcemap");
+const isMinify = process.argv.includes("--minify");
+/**
+ * @type {import('esbuild').Plugin}
+ */
+const esbuildProblemMatcherPlugin = {
+  name: "esbuild-problem-matcher",
 
+  setup(build) {
+    build.onStart(() => {
+      console.log("[watch] build started");
+    });
+    build.onEnd((result) => {
+      result.errors.forEach(({ text, location }) => {
+        console.error(`✘ [ERROR] ${text}`);
+        console.error(
+          `    ${location.file}:${location.line}:${location.column}:`,
+        );
+      });
+      console.log("[watch] build finished");
+    });
+  },
+};
 await Promise.all([
-  buildExtension(isWatch, isSourcemap),
-  buildWebview(isWatch, isSourcemap),
+  buildExtension(isWatch, isSourcemap, isMinify),
+  buildWebview(isWatch, isSourcemap, isMinify),
   buildSentrySourceMap(),
 ]);

download-lspjs.sh

@@ -1,12 +1,16 @@
 #!/usr/bin/env bash
 set -eu
 # Check if lspjs exists and if its a symlink then exit
-if [ -L lspjs ]; then
+if [ -L dist/lspjs ]; then
     echo "lspjs symlink exists, not downloading as you are most likely using a local version"
     exit 0
 fi
-mkdir -p lspjs/dist
-for var in "$@"
+mkdir -p dist/lspjs
+SEMGREP_VERSION=$(cat ./semgrep-version)
+echo "Downloading lspjs from S3 for version $SEMGREP_VERSION"
+for filename in Main.bc.js language-server-wasm.js semgrep-lsp-bindings.js semgrep-lsp.js
 do
-    curl https://static.semgrep.dev/static/turbo/$(cat ./semgrep-version)/language_server/dist/$var -o ./lspjs/dist/$var
+    echo "Downloading $filename"
+    curl "https://static.semgrep.dev/static/turbo/$SEMGREP_VERSION/language_server/dist/$filename" -o "./dist/lspjs/$filename"
 done
+echo "Downloaded lspjs"

download-osemgrep-pro.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -eu
+uname=$1
+case "${uname}" in
+    linux-x64*)    machine=manylinux;;
+    linux-arm64*)   machine=linux-arm64;;
+    darwin-x64*)   machine=osx;;
+    darwin-arm64)    machine=osx-m1;;
+    *)    machine=manylinux;;
+esac
+# NOT the same as the semgrep version!!!!
+OSEMGREP_PRO_VERSION=$(cat ./osemgrep-pro-version)
+BINARY=semgrep-core-proprietary-${machine}-${OSEMGREP_PRO_VERSION}
+# Check if osemgrep-pro exists and if its a symlink then exit
+if [ -L dist/osemgrep-pro ]; then
+    echo "osemgrep-pro symlink exists, not downloading as you are most likely using a local version"
+    exit 0
+fi
+mkdir -p dist
+echo "Downloading osemgrep-pro binary from S3 for version ${machine}-${OSEMGREP_PRO_VERSION}"
+aws s3 cp s3://deep-semgrep-artifacts/${BINARY} dist/osemgrep-pro
+echo "Downloaded osemgrep-pro binary"
+echo "Making osemgrep-pro binary executable"
+chmod +x dist/osemgrep-pro