refactor build process, build onefile, upgrade to v2.1.0, add homebrew release

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build with pyinstaller
+name: Build / Release
 
 on:
   pull_request:
@@ -7,44 +7,175 @@ on:
   push:
     branches:
       - main
+    tags:
+      - '*'
+  workflow_dispatch:
 
 jobs:
-  release:
-    runs-on: ${{ matrix.os }}
-
+  build:
     strategy:
       matrix:
-        os: [macos-10.15, windows-latest, ubuntu-latest]
+        include:
+          - runner: macos-11
+            os: darwin
+            arch: amd64
+          - runner: windows-2019
+            os: windows
+            arch: amd64
+          - runner: ubuntu-20.04
+            os: linux
+            arch: amd64
+          - runner: buildjet-2vcpu-ubuntu-2204-arm
+            os: linux
+            arch: arm64
 
+    runs-on: ${{ matrix.runner }}
+    outputs:
+      cli_version: ${{ steps.cli_version.outputs.cli_version }}
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
+
+      - name: Setup Python (GitHub Runner)
+        if: ${{ !contains(matrix.runner, 'buildjet') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10.11'
+
+      - name: Setup Python (BuildJet Runner)
+        if: contains(matrix.runner, 'buildjet')
+        uses: gabrielfalcao/pyenv-action@v14
+        with:
+          default: '3.10.11'
 
       - name: Create virtual environment
         run: make venv
 
       - name: Build using pyinstaller
-        run: make build
+        shell: bash
+        run: make clean all
 
-      - name: Smoke test
+      - name: Setup Docker on MacOS
+        if: matrix.os == 'darwin'
         run: |
-          ls dist/localstack
-          dist/localstack/localstack --help
-          dist/localstack/localstack config show
+          brew install docker
+          colima start
+
+      - name: Non-Docker Smoke tests
+        shell: bash
+        run: |
+          ls dist-bin/
+          cd dist-bin
+          # show the help
+          ./localstack --help
+          # show the config
+          ./localstack config show
+
+      - name: Community Docker Smoke tests (Linux, MacOS)
+        shell: bash
+        # GitHub Windows runner cannot run Docker containers (https://github.com/orgs/community/discussions/25491)
+        # Skip these checks for forks (forks do not have access to the LocalStack Pro API key)
+        if: matrix.os != 'windows' && !github.event.pull_request.head.repo.fork
+        run: |
+          # Pull images to avoid making smoke tests vulnerable to system behavior (docker pull speed)
+          docker pull localstack/localstack
+          cd dist-bin
+          # start community
+          ./localstack start -d
+          ./localstack wait -t 60
+          ./localstack status services --format plain
+          ./localstack status services --format plain | grep "s3=available"
+          ./localstack stop
+
+      - name: Pro Docker Smoke tests (Linux, MacOS)
+        shell: bash
+        # GitHub Windows runner cannot run Docker containers (https://github.com/orgs/community/discussions/25491)
+        # Skip these checks for forks (forks do not have access to the LocalStack Pro API key)
+        if: matrix.os != 'windows' && !github.event.pull_request.head.repo.fork
+        run: |
+          # Pull images to avoid making smoke tests vulnerable to system behavior (docker pull speed)
+          docker pull localstack/localstack-pro
+          cd dist-bin
+          # start pro
+          LOCALSTACK_API_KEY=${{ secrets.TEST_LOCALSTACK_API_KEY }} ./localstack start -d
+          ./localstack wait -t 60
+          ./localstack status services --format plain
+          ./localstack status services --format plain | grep "xray=available"
+          ./localstack stop
 
       - name: Set CLI version output
         id: cli_version
+        shell: bash
+        run: |
+          dist-bin/localstack --version
+          echo "cli_version=$(dist-bin/localstack --version)" >> $GITHUB_OUTPUT
+
+      - name: Archive distribution (Linux, MacOS)
+        if: matrix.os != 'windows'
+        run: |
+          cd dist-bin/
+          tar -czf ${{github.event.repository.name}}-${{steps.cli_version.outputs.cli_version}}-${{ matrix.os }}-${{ matrix.arch }}-onefile.tar.gz localstack
+          rm localstack
+          cd ../dist-dir/
+          tar -czf ${{github.event.repository.name}}-${{steps.cli_version.outputs.cli_version}}-${{ matrix.os }}-${{ matrix.arch }}.tar.gz localstack
+          rm -r localstack
+
+      - name: Archive distribution (Windows)
+        if: matrix.os == 'windows'
+        run: |
+          cd dist-bin/
+          Compress-Archive localstack.exe ${{github.event.repository.name}}-${{steps.cli_version.outputs.cli_version}}-${{ matrix.os }}-${{ matrix.arch }}.zip
+          rm localstack.exe
+          cd ../dist-dir/
+          Compress-Archive localstack ${{github.event.repository.name}}-${{steps.cli_version.outputs.cli_version}}-${{ matrix.os }}-${{ matrix.arch }}-onefile.zip
+          rm -r localstack
+
+      - name: Upload binary artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{github.event.repository.name}}-${{steps.cli_version.outputs.cli_version}}-${{ matrix.os }}-${{ matrix.arch }}-onefile
+          path: 'dist-bin/*'
+
+      - name: Upload folder artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{github.event.repository.name}}-${{steps.cli_version.outputs.cli_version}}-${{ matrix.os }}-${{ matrix.arch }}
+          path: 'dist-dir/*'
+
+  release:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+      - build
+    permissions:
+      contents: write
+    steps:
+      - name: Download Builds
+        uses: actions/download-artifact@v3
+        with:
+          path: builds
+
+      # TODO remove this section once we have native darwin arm64 builds
+      # This step is currently necessary for the homebrew action to pick up the MacOS AMD64 package for MacOS ARM64 / M1
+      # GitHub Roadmap: https://github.com/github/roadmap/issues/528
+      - name: (Intermediate) Use Darwin AMD64 for Darwin ARM64
         run: |
-          dist/localstack/localstack --version
-          echo "::set-output name=number::$(dist/localstack/localstack --version)"
+          cp ./builds/${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-darwin-amd64/* ./builds/${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-darwin-amd64/${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-darwin-arm64.tar.gz
+          cp ./builds/${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-darwin-amd64-onefile/* ./builds/${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-darwin-amd64-onefile/${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-darwin-arm64-onefile.tar.gz
 
-      - name: Archive distribution
+      - name: Generate Checksums
         run: |
-          cd dist/
-          tar -czf localstack-cli-${{steps.cli_version.outputs.number}}-${{ matrix.os }}.tar.gz localstack
+          # move all files from the builds subdirectories to the builds root folder
+          find ./builds/ -type f -print0 | xargs -0 mv -t ./builds/
+          # remove all (empty) subdirectories
+          find ./builds/ -mindepth 1 -maxdepth 1 -type d -print0 | xargs -r0 rm -R
+          # generate the checksums
+          cd builds
+          sha256sum *.{tar.gz,zip} > ${{github.event.repository.name}}-${{needs.build.outputs.cli_version}}-checksums.txt
 
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v2
+      - name: Release
+        uses: softprops/action-gh-release@v1
         with:
-          name: localstack-cli-${{steps.cli_version.outputs.number}}-${{ matrix.os }}
-          path: dist/localstack-cli-${{steps.cli_version.outputs.number}}-${{ matrix.os }}.tar.gz
+          files: 'builds/*'
+          draft: true
+          token: ${{ secrets.LOCALSTACK_GITHUB_TOKEN }}

.github/workflows/homebrew.yml

@@ -0,0 +1,60 @@
+name: Release Homebrew Tap
+
+on:
+  release:
+    # Start Homebrew Releaser when a new GitHub release of the CLI package is _published_
+    types: [published]
+
+jobs:
+  homebrew-releaser:
+    runs-on: ubuntu-latest
+    name: homebrew-releaser
+    steps:
+      - name: Add published release to Homebrew Tap
+        uses: Justintime50/homebrew-releaser@v1
+        with:
+          # The name of the homebrew tap to publish your formula to as it appears on GitHub.
+          # Required - strings
+          homebrew_owner: localstack
+          homebrew_tap: homebrew-tap
+
+          # Logs debugging info to console.
+          # Default is shown - boolean
+          debug: true
+
+          # The name of the folder in your homebrew tap where formula will be committed to.
+          # Default is shown - string
+          formula_folder: Formula
+
+          # The Personal Access Token (saved as a repo secret) that has `repo` permissions for the repo running the action AND Homebrew tap you want to release to.
+          # Required - string
+          github_token: ${{ secrets.LOCALSTACK_GITHUB_TOKEN }}
+
+
+          # Git author info used to commit to the homebrew tap.
+          # Defaults are shown - strings
+          commit_owner: localstack-bot
+          commit_email: 88328844+localstack-bot@users.noreply.github.com
+
+          # Custom install command for your formula.
+          # Required - string
+          # The indentation is on purpose to fix the multiline indentation in the final formula
+          install: |
+            libexec.install Dir["*"]
+                bin.install_symlink libexec/"localstack"
+
+          # Custom test command for your formula so you can run `brew test`.
+          # Optional - string
+          test: |
+            assert_match /LocalStack Command Line Interface/, shell_output("#{bin}/localstack --help", 0)
+
+          # Adds URL and checksum targets for different OS and architecture pairs. Using this option assumes
+          # a tar archive exists on your GitHub repo with the following URL pattern (this cannot be customized):
+          # https://github.com/{GITHUB_OWNER}/{REPO_NAME}/releases/download/{TAG}/{REPO_NAME}-{VERSION}-{OPERATING_SYSTEM}-{ARCHITECTURE}.tar.gz'
+          # Darwin AMD pre-existing path example: https://github.com/justintime50/myrepo/releases/download/v1.2.0/myrepo-1.2.0-darwin-amd64.tar.gz
+          # Linux ARM pre-existing path example: https://github.com/justintime50/myrepo/releases/download/v1.2.0/myrepo-1.2.0-linux-arm64.tar.gz
+          # Optional - booleans
+          target_darwin_amd64: true
+          target_darwin_arm64: true
+          target_linux_amd64: true
+          target_linux_arm64: true

.gitignore

@@ -1,3 +1,8 @@
+# Build dist folders
+dist-bin
+dist-dir
+
+# IntelliJ
 .idea/
 *.iml
 *~

Makefile

@@ -1,5 +1,6 @@
 VENV_BIN = python3 -m venv
 VENV_DIR ?= .venv
+PYINSTALLER_ARGS = --distpath=dist-bin --onefile
 
 ifeq ($(OS), Windows_NT)
 	VENV_ACTIVATE = $(VENV_DIR)/Scripts/activate
@@ -9,7 +10,7 @@ endif
 
 VENV_RUN = . $(VENV_ACTIVATE)
 
-all: venv build
+all: dist-bin/localstack dist-dir/localstack
 
 venv: $(VENV_ACTIVATE)
 
@@ -19,23 +20,22 @@ $(VENV_ACTIVATE): requirements.txt
 	$(VENV_RUN); pip install -r requirements.txt
 	touch $(VENV_ACTIVATE)
 
-# currently botocore and boto3 are required because patch.py of localstack-client>=1.33
-# once that is remedied, we can exclude boto3 and botocore modules again
-dist/localstack/localstack: main.py
+dist-bin/localstack build: $(VENV_ACTIVATE) main.py
 	$(VENV_RUN); pyinstaller main.py \
 		$(PYINSTALLER_ARGS) -n localstack \
-		--exclude-module moto \
-		--hidden-import docker
-	rm -rf dist/localstack/botocore/data
+		--hidden-import localstack_ext.cli.localstack \
+		--additional-hooks-dir hooks
 
-build: venv dist/localstack/localstack
+dist-dir/localstack: PYINSTALLER_ARGS=--distpath=dist-dir
+dist-dir/localstack: $(VENV_ACTIVATE) main.py build
 
 clean:
 	rm -rf build/
-	rm -rf dist/
+	rm -rf dist-bin/
+	rm -rf dist-dir/
 
 clean-venv:
 	rm -rf $(VENV_DIR)
 
-.PHONY: clean clean-venv
+.PHONY: all build clean clean-venv
 

README.md

@@ -1,34 +1,56 @@
-localstack-packaged-cli
+LocalStack CLI
 =======================
 
-Repository for the build config that packages the localstack cli into a standalone binary.
-
-## Build
-
+This repository contains building instructions for binary builds of the LocalStack CLI.
+It does not contain the actual source for the CLI, since the LocalStack CLI is basically just the Python package `localstack` (published on PyPi) with it's install dependencies (and without any extras).
+This is why this repository just contains the build config and pipeline that packages the LocalStack CLI python package into a standalone binary using PyInstaller.
+
+## Creating a Release
+In order to create a release, just perform the following tasks:
+- Create a commit which sets a new explicit version for `localstack` in the `requirements.txt`.
+  - For example: `localstack==2.1.0`
+- Create a tag for the commit: `v<version>`.
+  - For example: `git tag v2.1.0`
+- Push the tag (`git push origin v<version>`)
+- This will trigger the following actions:
+  - The tag will trigger the ["Build / Release"](.github/workflows/build.yml) GitHub workflow.
+  - It will build the binaries for the different systems and create a GitHub release draft.
+- Publish the GitHub release draft.
+- This will trigger the ["Release Homebrew Tap"](.github/workflows/homebrew.yml) GitHub workflow.
+  - It will take the release artifacts and update the Homebrew formula in [localstack/homebrew-tap](https://github.com/localstack/homebrew-tap).
+
+### Dev Releases
+If a dev release is created, the tag name has to have the same name as the version of `localstack-core` being used (because this is the output of `localstack --version`).
+Otherwise, the ["Release Homebrew Tap"](.github/workflows/homebrew.yml) GitHub workflow will not be able to find the artifacts.
+
+## Manual Build
 ### python3-dev
-
 You need Python developer version libraries in your path to be able to build the distribution.
-
-For most of us who use pyenv, this is done with
-
-```bash
-pyenv install 3.8-dev
-pyenv local 3.8-dev
+For most of us who use pyenv, this is done with:
+- MacOS:
+  ```bash
+  env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install 3.10-dev
+  ```
+- Linux:
+  ```bash
+  env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.10-dev
+  ```
+
+Activate the version:
+```
+pyenv local 3.10-dev
 python --version
 ```
+This should print something like `Python 3.10.11+`.
 
-should print something like `Python 3.8.15+`.
-
-### make all
-
-Just run
-
+### Building
+You can build the specific versions by calling the respective make target:
 ```bash
+make clean dist-bin/localstack
+# or:
+make clean dist-dir/localstack
+# or both:
 make clean all
 ```
-
-in `dist/localstack` you should now find the binary assets.
-
-If you want a single binary you can run `PYINSTALLER_ARGS=-F make clean all`.
-This will create a single binary `dist/localstack`.
+You can find the binary assets in `dist-bin/` and `dist-dir`.
 The single binary has a slower startup time than the binary distribution.