feat: determine, inject and publish the LS protocol version [HEAD-692] (

#4853)

* feat: add test for protocol version output

* feat: inject protocol version into cli during build

* feat: add protocol version to upload

* fix: copy fips file when creating artifacts, not from the makefile

because else, during the build, we're creating & copying unnecessary dirs & files

* fix: pr suggestions, upload-artifacts.sh

* fix: workflow

* fix: try to fix windows acceptance test

.circleci/Dockerfile

@@ -19,6 +19,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3 \
     python3-pip \
     python-is-python3 \
+    python3-requests \
+    python3-yaml \
     pipenv \
     apt-transport-https \
     curl \
@@ -67,7 +69,6 @@ ENV PATH=/home/circleci/.sdkman/candidates/scala/current/bin:/home/circleci/.sdk
 # install dotnet
 RUN curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin --channel 7.0
 ENV PATH=/home/circleci/.dotnet:$PATH
-
 ENV TEMP=/tmp
 ENV TMP=$TEMP
 ENV TMPDIR=$TEMP

.circleci/config.yml

@@ -38,12 +38,12 @@ executors:
       - image: alpine:3.17
   docker-amd64:
     docker:
-      - image: bastiandoetsch209/cli-build:20230815-154734
+      - image: bastiandoetsch209/cli-build:20230912-103845
     working_directory: /mnt/ramdisk/snyk
     resource_class: large
   docker-arm64:
     docker:
-      - image: bastiandoetsch209/cli-build-arm64:20230815-154734
+      - image: bastiandoetsch209/cli-build-arm64:20230912-103845
     working_directory: /mnt/ramdisk/snyk
     resource_class: arm.large
   linux-ubuntu-mantic-amd64:
@@ -181,6 +181,8 @@ commands:
           key: chocolatey-cache-v3-{{ arch }}-{{ checksum ".circleci/chocolatey.config" }}
           paths:
             - ~\AppData\Local\Temp\chocolatey
+      - install-deps-python:
+          os: win
 
   install-deps-windows-make:
     steps:
@@ -198,6 +200,8 @@ commands:
           key: chocolatey-cache-v3-{{ arch }}-{{ checksum ".circleci/chocolatey.config" }}
           paths:
             - ~\AppData\Local\Temp\chocolatey
+      - install-deps-python:
+          os: win
 
   install-deps-windows-node:
     steps:
@@ -224,6 +228,35 @@ commands:
           name: No dependencies to install
           command: echo all done!
 
+  install-deps-python:
+    parameters:
+      os:
+        type: string
+        default: linux
+      modules:
+        type: string
+        default: requests PyYAML
+    steps:
+      - when:
+          condition:
+            matches:
+              pattern: '^(linux.*|macos.*|alpine.*|docker.*)'
+              value: << parameters.os >>
+          steps:
+            - run:
+                name: Installing python modules
+                command: python3 -m pip install << parameters.modules >>
+      - when:
+          condition:
+            matches:
+              pattern: '^(win.*|cbl-mariner.*)'
+              value: << parameters.os >>
+          steps:
+            - run:
+                shell: bash.exe
+                name: Installing python modules
+                command: python -m pip install << parameters.modules >>
+
   install-deps-macos-full:
     parameters:
       items:
@@ -240,14 +273,25 @@ commands:
           key: acceptance-tests-macos-<< parameters.items >>
           paths:
             - ~/Library/Caches/Homebrew
+      - install-deps-python:
+          os: macos
+
+  install-deps-macos-build:
+    parameters:
+      items:
+        type: string
+        default: go gradle python elixir composer gradle@6 maven sbt dotnet
+    steps:
+      - install-deps-python:
+          os: macos
 
   install-deps-alpine-full:
     steps:
       - run:
           name: Installing Node.js + other test dependencies
           command: |
             apk add --update nodejs npm bash maven git go gradle python3 py3-pip elixir composer
-            pip3 install pipenv
+            pip3 install pipenv requests PyYAML
 
   failed-release-notification:
     steps:
@@ -350,6 +394,7 @@ workflows:
           go_arch: amd64
           go_download_base_url: << pipeline.parameters.go_download_base_url >>
           executor: macos-amd64
+          install_deps_extension: macos-build
           requires:
             - prepare-build
 
@@ -921,6 +966,11 @@ jobs:
       - run:
           name: Validating artifacts
           command: ./release-scripts/validate-checksums.sh
+      - run:
+          name: Generate Protocol Version File
+          command: |
+            make ls-protocol-metadata
+            cp binary-releases/ls-protocol-version-* binary-releases/fips/
       - store_artifacts:
           path: ./binary-releases
       - persist_to_workspace:
@@ -934,6 +984,8 @@ jobs:
             - binary-releases/fips/snyk-*.sha256
             - binary-releases/RELEASE_NOTES.md
             - binary-releases/snyk.tgz
+            - binary-releases/ls-protocol-version-*
+            - binary-releases/fips/ls-protocol-version-*
 
   pre-release:
     executor: docker-amd64

Makefile

@@ -5,12 +5,14 @@
 # Documentation: https://www.gnu.org/software/make/manual/make.html
 #
 
+#use bash instead of sh
+export SHELL=/bin/bash
 WORKING_DIR = $(CURDIR)
 PKG := npx pkg ./ --compress Brotli
 BINARY_WRAPPER_DIR = ts-binary-wrapper
 EXTENSIBLE_CLI_DIR = cliv2
 BINARY_RELEASES_FOLDER_TS_CLI = binary-releases
-BINARY_OUTPUT_FOLDER = binary-releases
+export BINARY_OUTPUT_FOLDER = binary-releases
 SHASUM_CMD = shasum
 GOHOSTOS = $(shell go env GOHOSTOS)
 export PYTHON = python
@@ -259,4 +261,9 @@ release-pre:
 format:
 	@echo "-- Formatting code"
 	@npm run format
-	@pushd cliv2; $(MAKE) format; popd
+	@pushd $(EXTENSIBLE_CLI_DIR); $(MAKE) format; popd
+
+.PHONY: ls-protocol-metadata
+ls-protocol-metadata: $(BINARY_RELEASES_FOLDER_TS_CLI)/version
+	@echo "-- Generating protocol metadata"
+	@pushd $(EXTENSIBLE_CLI_DIR) && $(MAKE) generate-ls-protocol-metadata bindir=$(WORKING_DIR)/$(BINARY_OUTPUT_FOLDER) && popd

cliv2/Makefile

@@ -15,6 +15,7 @@ CLI_V1_LOCATION =
 USE_LEGACY_EXECUTABLE_NAME =
 LDFLAGS = -s -w
 SHASUM_CMD = shasum
+export LS_PROTOCOL_VERSION=
 
 # Make directories per convention
 prefix = /usr/local
@@ -139,14 +140,13 @@ summary:
 	@echo "$(LOG_PREFIX)  Building version:             $(CLI_V1_VERSION_TAG)"
 	@echo "$(LOG_PREFIX)  LS version:                   $(LS_COMMIT_HASH)"
 
-
 # prepare the workspace and cache global parameters
 .PHONY: configure
 configure: summary $(CACHE_DIR) $(CACHE_DIR)/variables.mk $(V1_DIRECTORY)/$(V1_EMBEDDED_FILE_OUTPUT) dependencies $(CACHE_DIR)/prepare-3rd-party-licenses
 
-$(BUILD_DIR)/$(V2_EXECUTABLE_NAME): $(BUILD_DIR) $(SRCS)
+$(BUILD_DIR)/$(V2_EXECUTABLE_NAME): $(BUILD_DIR) $(SRCS) generate-ls-protocol-metadata
 	@echo "$(LOG_PREFIX) Building ( $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) )"
-	@GOEXPERIMENT=$(FIPS_CRYPTO_BACKEND) GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -tags=application -ldflags="$(LDFLAGS) -X github.com/snyk/snyk-ls/application/config.Version=$(LS_COMMIT_HASH)  -X main.internalOS=$(GOOS) -X github.com/snyk/cli/cliv2/internal/embedded/cliv1.snykCLIVersion=$(CLI_V1_VERSION_TAG)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go
+	@GOEXPERIMENT=$(FIPS_CRYPTO_BACKEND) GOOS=$(_GO_OS) GOARCH=$(GOARCH) $(GOCMD) build -tags=application -ldflags="$(LDFLAGS) -X github.com/snyk/snyk-ls/application/config.Version=$(LS_COMMIT_HASH) -X github.com/snyk/snyk-ls/application/config.LsProtocolVersion=$(LS_PROTOCOL_VERSION)  -X main.internalOS=$(GOOS) -X github.com/snyk/cli/cliv2/internal/embedded/cliv1.snykCLIVersion=$(CLI_V1_VERSION_TAG)" -o $(BUILD_DIR)/$(V2_EXECUTABLE_NAME) $(WORKING_DIR)/cmd/cliv2/main.go
 
 .PHONY: fips
 fips:
@@ -208,12 +208,20 @@ $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME):
 	@echo "$(LOG_PREFIX) Building legacy CLI"
 	@cd $(V1_WORKING_DIR) && npm i && npm run $(V1_BUILD_TYPE)
 	@$(MAKE) -C $(V1_WORKING_DIR) $(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME) BINARY_RELEASES_FOLDER_TS_CLI=$(V1_BINARY_FOLDER)
-	
+
 .PHONY: build-ts-cli
 build-ts-cli: $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME)
 	$(eval CLI_V1_VERSION_TAG := $(shell cat $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/version))
 	$(eval CLI_V1_LOCATION := $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/)
-
+
+.PHONY: generate-ls-protocol-metadata
+generate-ls-protocol-metadata:
+	$(eval CLI_V1_VERSION_TAG := $(shell cat $(DESTDIR)$(bindir)/version))
+	@echo "$(LOG_PREFIX) Determining LS protocol version"
+	@echo "$(LOG_PREFIX) Writing protocol version file to $(DESTDIR)$(bindir)"
+	$(eval LS_PROTOCOL_VERSION := $(shell $(PYTHON) $(WORKING_DIR)/../release-scripts/write-ls-protocol-version.py $(LS_COMMIT_HASH) $(CLI_V1_VERSION_TAG) $(DESTDIR)$(bindir)))
+	@echo "$(LOG_PREFIX) LS protocol version: $(LS_PROTOCOL_VERSION)"
+
 .PHONY: clean-ts-cli
 clean-ts-cli:
 	@echo "$(LOG_PREFIX) Cleaning legacy CLI"

cliv2/go.mod

@@ -12,10 +12,10 @@ require (
 	github.com/snyk/cli-extension-dep-graph v0.0.0-20230831101913-402a467e32e7
 	github.com/snyk/cli-extension-iac-rules v0.0.0-20230601153200-c572cfce46ce
 	github.com/snyk/cli-extension-sbom v0.0.0-20230831113416-7ffac8738181
-	github.com/snyk/go-application-framework v0.0.0-20230825084328-b839e0e50201
+	github.com/snyk/go-application-framework v0.0.0-20230911124155-af80929f69be
 	github.com/snyk/go-httpauth v0.0.0-20230726132335-d454674305a7
 	github.com/snyk/snyk-iac-capture v0.6.0
-	github.com/snyk/snyk-ls v0.0.0-20230901155643-31770bf57616
+	github.com/snyk/snyk-ls v0.0.0-20230911152618-39fc0f68d431
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
@@ -37,7 +37,7 @@ require (
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/apparentlymart/go-versions v1.0.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go v1.44.122 // indirect
+	github.com/aws/aws-sdk-go v1.45.6 // indirect
 	github.com/aymanbagabas/go-osc52 v1.2.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bmatcuk/doublestar v1.3.4 // indirect