Add workflow to build executables from python scripts

.github/pytools/Sign-File.ps1

@@ -0,0 +1,78 @@
+[CmdletBinding()]
+param (
+    [Parameter()]
+    [String]
+    $Path
+)
+
+
+function FindSignTool {
+    $SignTool = "signtool.exe"
+    if (Get-Command $SignTool -ErrorAction SilentlyContinue) {
+        return $SignTool
+    }
+    $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x64\signtool.exe"
+    if (Test-Path -Path $SignTool -PathType Leaf) {
+        return $SignTool
+    }
+    $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x86\signtool.exe"
+    if (Test-Path -Path $SignTool -PathType Leaf) {
+        return $SignTool
+    }
+    $sdkVers = "10.0.22000.0", "10.0.20348.0", "10.0.19041.0", "10.0.17763.0"
+    Foreach ($ver in $sdkVers)
+    {
+        $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\${ver}\x64\signtool.exe"
+        if (Test-Path -Path $SignTool -PathType Leaf) {
+            return $SignTool
+        }
+    }
+    "signtool.exe not found"
+    Exit 1
+}
+
+function SignEsptool {
+    param(
+        [Parameter()]
+        [String]
+        $Path
+    )
+
+    $SignTool = FindSignTool
+    "Using: $SignTool"
+    $CertificateFile = [system.io.path]::GetTempPath() + "certificate.pfx"
+
+    if ($null -eq $env:CERTIFICATE) {
+        "CERTIFICATE variable not set, unable to sign the file"
+        Exit 1
+    }
+
+    if ("" -eq $env:CERTIFICATE) {
+        "CERTIFICATE variable is empty, unable to sign the file"
+        Exit 1
+    }
+
+    $SignParameters = @("sign", "/tr", 'http://timestamp.digicert.com', "/td", "SHA256", "/f", $CertificateFile, "/fd", "SHA256")
+    if ($env:CERTIFICATE_PASSWORD) {
+        "CERTIFICATE_PASSWORD detected, using the password"
+        $SignParameters += "/p"
+        $SignParameters += $env:CERTIFICATE_PASSWORD
+    }
+    $SignParameters += $Path
+
+    [byte[]]$CertificateBytes = [convert]::FromBase64String($env:CERTIFICATE)
+    [IO.File]::WriteAllBytes($CertificateFile, $CertificateBytes)
+
+    &$SignTool $SignParameters
+
+    if (0 -eq $LASTEXITCODE) {
+        Remove-Item $CertificateFile
+    } else {
+        Remove-Item $CertificateFile
+        "Signing failed"
+        Exit 1
+    }
+
+}
+
+SignEsptool ${Path}

.github/scripts/upload_py_tools.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+CHANGED_FILES=$1
+echo "Pushing '$CHANGED_FILES' as $GITHUB_ACTOR"
+git config --global github.user "$GITHUB_ACTOR"
+git config --global user.name "$GITHUB_ACTOR"
+git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com"
+for tool in $CHANGED_FILES; do
+	git add tools/$tool.exe
+done
+git commit -m "Push binary to tools"
+git push

.github/workflows/build_py_tools.yml

@@ -0,0 +1,141 @@
+name: Build Python Tools
+
+on:
+  pull_request:
+    paths:
+    - 'tools/get.py'
+    - 'tools/espota.py'
+    - 'tools/gen_esp32part.py'
+    - 'tools/gen_insights_package.py'
+
+jobs:
+  find-changed-tools:
+    name: Check if tools have been changed
+    runs-on: ubuntu-20.04
+    outputs:
+      any_changed: ${{ steps.verify-changed-files.outputs.any_changed }}
+      all_changed_files: ${{ steps.verify-changed-files.outputs.all_changed_files }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Verify Python Tools Changed
+        uses: tj-actions/changed-files@v36
+        id: verify-changed-files
+        with:
+          fetch_depth: '2'
+          since_last_remote_commit: 'true'
+          files: |
+            tools/get.py
+            tools/espota.py
+            tools/gen_esp32part.py
+            tools/gen_insights_package.py
+      - name: List all changed files
+        shell: bash
+        run: |
+          for file in ${{ steps.verify-changed-files.outputs.all_changed_files }}; do
+            echo "$file was changed"
+          done
+
+  build-pytools-binaries:
+    name: Build python tools binaries for ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: find-changed-tools
+    if: needs.find-changed-tools.outputs.any_changed == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest, ubuntu-20.04, ARM, ARM64]
+        include:
+        - os: windows-latest
+          TARGET: win64
+          EXTEN: .exe
+          SEPARATOR: ';'
+        - os: macos-latest
+          TARGET: macos
+          SEPARATOR: ':'
+        - os: ubuntu-20.04
+          TARGET: linux-amd64
+          SEPARATOR: ':'
+        - os: ARM
+          CONTAINER: python:3.8-bullseye
+          TARGET: arm
+          SEPARATOR: ':'
+        - os: ARM64
+          CONTAINER: python:3.8-bullseye
+          TARGET: arm64
+          SEPARATOR: ':'
+    container: ${{ matrix.CONTAINER }} # use python container on ARM
+    env:
+      DISTPATH: pytools-${{ matrix.TARGET }}
+      PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi"
+    steps:
+      - name: List changed tools
+        shell: bash
+        run: |
+          CHANGED_FILES=()
+          for file in ${{ needs.find-changed-tools.outputs.all_changed_files }}; do
+              file="${file#*\/}"
+              file="${file%\.*}"
+              CHANGED_FILES+=("$file")
+          done
+          CHANGED_FILES="${CHANGED_FILES[@]}"
+          echo "CHANGED_TOOLS=$CHANGED_FILES" >> "$GITHUB_ENV"
+          for tool in ${{ env.CHANGED_TOOLS }}; do
+            echo "tool $tool was changed"
+          done
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Set up Python 3.8
+        # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
+        if: matrix.os != 'ARM' && matrix.os != 'ARM64'
+        uses: actions/setup-python@master
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyinstaller requests
+      - name: Build with PyInstaller
+        shell: bash
+        run: |
+          for tool in ${{ env.CHANGED_TOOLS }}; do
+            pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=.github/pytools/espressif.ico tools/$tool.py
+          done
+      - name: Sign binaries
+        if: matrix.os == 'windows-latest'
+        env:
+          CERTIFICATE: ${{ secrets.CERTIFICATE }}
+          CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
+        shell: pwsh
+        run: |
+          $data = Write-Output ${{ env.CHANGED_TOOLS }}
+          foreach ( $node in $data )
+          {
+            ./.github/pytools/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/$node.exe
+          }
+      - name: Test binaries
+        shell: bash
+        run: |
+          for tool in ${{ env.CHANGED_TOOLS }}; do
+            ./${{ env.DISTPATH }}/$tool${{ matrix.EXTEN }} -h
+          done
+      - name: Push binary to tools
+        if: matrix.os == 'windows-latest'
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        shell: bash
+        run: |
+          for tool in ${{ env.CHANGED_TOOLS }}; do
+            cp -f ./${{ env.DISTPATH }}/$tool.exe tools/$tool.exe
+          done
+          bash .github/scripts/upload_py_tools.sh "${{ env.CHANGED_TOOLS }}"
+      - name: Archive artifact
+        uses: actions/upload-artifact@master
+        with:
+          name: ${{ env.DISTPATH }}
+          path: ${{ env.DISTPATH }}

tools/get.py

@@ -35,7 +35,13 @@
 if 'Windows' in platform.system():
     import requests
 
-current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))
+# determine if application is a script file or frozen exe
+if getattr(sys, 'frozen', False):
+    current_dir = os.path.dirname(os.path.realpath(unicode(sys.executable)))
+elif __file__:
+    current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))
+
+#current_dir = os.path.dirname(os.path.realpath(unicode(__file__)))
 dist_dir = current_dir + '/dist/'
 
 def sha256sum(filename, blocksize=65536):
@@ -88,6 +94,8 @@ def unpack(filename, destination):
 
     # a little trick to rename tool directories so they don't contain version number
     rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-')
+    if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'):
+        rename_to = 'esp32-arduino-libs'
     if rename_to != dirname:
         print('Renaming {0} to {1} ...'.format(dirname, rename_to))
         if os.path.isdir(rename_to):
@@ -217,10 +225,16 @@ def identify_platform():
     return arduino_platform_names[sys_name][bits]
 
 if __name__ == '__main__':
+    is_test = (len(sys.argv) > 1 and sys.argv[1] == '-h')
+    if is_test:
+        print('Test run!')
     identified_platform = identify_platform()
     print('Platform: {0}'.format(identified_platform))
     tools_to_download = load_tools_list(current_dir + '/../package/package_esp32_index.template.json', identified_platform)
     mkdir_p(dist_dir)
     for tool in tools_to_download:
-        get_tool(tool)
+        if is_test:
+            print('Would install: {0}'.format(tool['archiveFileName']))
+        else:
+            get_tool(tool)
     print('Platform Tools Installed')