Release v2.0.18 with ESP-IDF v4.4.8

.github/scripts/merge_packages.py

@@ -5,7 +5,8 @@
 # Written by Ivan Grokhotkov, 2015
 #
 from __future__ import print_function
-from distutils.version import LooseVersion
+#from distutils.version import LooseVersion
+from packaging.version import Version
 import re
 import json
 import sys
@@ -33,7 +34,7 @@ def merge_objects(versions, obj):
 def pkgVersionNormalized(versionString):
 
     verStr = str(versionString)
-    verParts = re.split('\.|-rc', verStr, flags=re.IGNORECASE)
+    verParts = re.split('\.|-rc|-alpha', verStr, flags=re.IGNORECASE)
 
     if len(verParts) == 3:
         if (sys.version_info > (3, 0)): # Python 3
@@ -74,7 +75,8 @@ def main(args):
             print("Adding platform {0}-{1}".format(name, version), file=sys.stderr)
             pkg1['platforms'].append(platforms[name][version])
 
-    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+    #pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: LooseVersion(pkgVersionNormalized(k['version'])), reverse=True)
+    pkg1['platforms'] = sorted(pkg1['platforms'], key=lambda k: Version(pkgVersionNormalized(k['version'])), reverse=True)
 
     json.dump({'packages':[pkg1]}, sys.stdout, indent=2)
 

.github/scripts/on-push.sh

@@ -4,12 +4,16 @@ set -e
 
 export ARDUINO_BUILD_DIR="$HOME/.arduino/build.tmp"
 
+# Relative path test
+export PATH="./bin:$PATH"
+
 function build(){
     local target=$1
     local fqbn=$2
     local chunk_index=$3
     local chunks_cnt=$4
-    shift; shift; shift; shift;
+    local build_log=$5
+    shift; shift; shift; shift; shift;
     local sketches=$*
 
     local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
@@ -22,6 +26,9 @@ function build(){
     if [ "$OS_IS_LINUX" == "1" ]; then
         args+=" -p $ARDUINO_ESP32_PATH/libraries"
         args+=" -i $chunk_index -m $chunks_cnt"
+        if [ $build_log -eq 1 ]; then
+            args+=" -l $build_log"
+        fi
         ${BUILD_SKETCHES} ${args}
     else
         for sketch in ${sketches}; do
@@ -45,6 +52,7 @@ fi
 
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
+BUILD_LOG=$3
 BUILD_PIO=0
 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
     CHUNK_INDEX=0
@@ -55,6 +63,10 @@ elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then
     BUILD_PIO=1
 fi
 
+if [ -z "$BUILD_LOG" ] || [ "$BUILD_LOG" -le 0 ]; then
+    BUILD_LOG=0
+fi
+
 #echo "Updating submodules ..."
 #git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
 
@@ -81,11 +93,25 @@ if [ "$BUILD_PIO" -eq 0 ]; then
       $ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino\
       $ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino\
     "
+    #create sizes_file
+    sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json"
 
-    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
-    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
-    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
-    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
+    if [ "$BUILD_LOG" -eq 1 ]; then
+        #create sizes_file and echo start of JSON array with "boards" key
+        echo "{\"boards\": [" > $sizes_file
+    fi
+
+    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32XX
+    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32XX
+    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+
+    if [ "$BUILD_LOG" -eq 1 ]; then
+        #remove last comma from the last JSON object
+        sed -i '$ s/.$//' "$sizes_file"
+        #echo end of JSON array
+        echo "]}" >> $sizes_file
+    fi
 else
     source ${SCRIPTS_DIR}/install-platformio-esp32.sh
     # PlatformIO ESP32 Test

.github/scripts/on-release.sh

@@ -36,6 +36,12 @@ echo "Event: $GITHUB_EVENT_NAME, Repo: $GITHUB_REPOSITORY, Path: $GITHUB_WORKSPA
 echo "Action: $action, Branch: $RELEASE_BRANCH, ID: $RELEASE_ID"
 echo "Tag: $RELEASE_TAG, Draft: $draft, Pre-Release: $RELEASE_PRE"
 
+# Try extracting something like a JSON with a "boards" array/element and "vendor" fields
+BOARDS=`echo $RELEASE_BODY | grep -Pzo '(?s){.*}' | jq -r '.boards[]? // .boards? // empty' | xargs echo -n 2>/dev/null`
+VENDOR=`echo $RELEASE_BODY | grep -Pzo '(?s){.*}' | jq -r '.vendor? // empty' | xargs echo -n 2>/dev/null`
+if ! [ -z "${BOARDS}" ]; then echo "Releasing board(s): $BOARDS" ; fi
+if ! [ -z "${VENDOR}" ]; then echo "Setting packager: $VENDOR" ; fi
+
 function get_file_size(){
     local file="$1"
     if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -171,12 +177,26 @@ mkdir -p "$PKG_DIR/tools"
 
 # Copy all core files to the package folder
 echo "Copying files for packaging ..."
-cp -f  "$GITHUB_WORKSPACE/boards.txt"                       "$PKG_DIR/"
+if [ -z "${BOARDS}" ]; then
+    # Copy all variants
+    cp -f  "$GITHUB_WORKSPACE/boards.txt"                   "$PKG_DIR/"
+    cp -Rf "$GITHUB_WORKSPACE/variants"                     "$PKG_DIR/"
+else
+    # Remove all entries not starting with any board code or "menu." from boards.txt
+    cat "$GITHUB_WORKSPACE/boards.txt" | grep "^menu\."         >  "$PKG_DIR/boards.txt"
+    for board in ${BOARDS} ; do
+        cat "$GITHUB_WORKSPACE/boards.txt" | grep "^${board}\." >> "$PKG_DIR/boards.txt"
+    done
+    # Copy only relevant variant files
+    mkdir "$PKG_DIR/variants/"
+    for variant in `cat ${PKG_DIR}/boards.txt | grep "\.variant=" | cut -d= -f2` ; do
+        cp -Rf "$GITHUB_WORKSPACE/variants/${variant}"      "$PKG_DIR/variants/"
+    done
+fi
 cp -f  "$GITHUB_WORKSPACE/package.json"                     "$PKG_DIR/"
 cp -f  "$GITHUB_WORKSPACE/programmers.txt"                  "$PKG_DIR/"
 cp -Rf "$GITHUB_WORKSPACE/cores"                            "$PKG_DIR/"
 cp -Rf "$GITHUB_WORKSPACE/libraries"                        "$PKG_DIR/"
-cp -Rf "$GITHUB_WORKSPACE/variants"                         "$PKG_DIR/"
 cp -f  "$GITHUB_WORKSPACE/tools/espota.exe"                 "$PKG_DIR/tools/"
 cp -f  "$GITHUB_WORKSPACE/tools/espota.py"                  "$PKG_DIR/tools/"
 cp -f  "$GITHUB_WORKSPACE/tools/gen_esp32part.py"           "$PKG_DIR/tools/"
@@ -196,17 +216,22 @@ find "$PKG_DIR" -name '*.git*' -type f -delete
 # Replace tools locations in platform.txt
 echo "Generating platform.txt..."
 cat "$GITHUB_WORKSPACE/platform.txt" | \
-sed "s/version=.*/version=$ver$extent/g" | \
-sed 's/tools.xtensa-esp32-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32-elf/tools.xtensa-esp32-elf-gcc.path=\{runtime.tools.xtensa-esp32-elf-gcc.path\}/g' | \
-sed 's/tools.xtensa-esp32s2-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32s2-elf/tools.xtensa-esp32s2-elf-gcc.path=\{runtime.tools.xtensa-esp32s2-elf-gcc.path\}/g' | \
-sed 's/tools.xtensa-esp32s3-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32s3-elf/tools.xtensa-esp32s3-elf-gcc.path=\{runtime.tools.xtensa-esp32s3-elf-gcc.path\}/g' | \
-sed 's/tools.riscv32-esp-elf-gcc.path={runtime.platform.path}\/tools\/riscv32-esp-elf/tools.riscv32-esp-elf-gcc.path=\{runtime.tools.riscv32-esp-elf-gcc.path\}/g' | \
-sed 's/tools.esptool_py.path={runtime.platform.path}\/tools\/esptool/tools.esptool_py.path=\{runtime.tools.esptool_py.path\}/g' | \
-sed 's/debug.server.openocd.path={runtime.platform.path}\/tools\/openocd-esp32\/bin\/openocd/debug.server.openocd.path=\{runtime.tools.openocd-esp32.path\}\/bin\/openocd/g' | \
-sed 's/debug.server.openocd.scripts_dir={runtime.platform.path}\/tools\/openocd-esp32\/share\/openocd\/scripts\//debug.server.openocd.scripts_dir=\{runtime.tools.openocd-esp32.path\}\/share\/openocd\/scripts\//g' | \
-sed 's/debug.server.openocd.scripts_dir.windows={runtime.platform.path}\\tools\\openocd-esp32\\share\\openocd\\scripts\\/debug.server.openocd.scripts_dir.windows=\{runtime.tools.openocd-esp32.path\}\\share\\openocd\\scripts\\/g' \
+sed "s/version=.*/version=$RELEASE_TAG/g" | \
+sed 's/{runtime\.platform\.path}.tools.xtensa-esp-elf-gdb/\{runtime.tools.xtensa-esp-elf-gdb.path\}/g' | \
+sed 's/{runtime\.platform\.path}.tools.xtensa-esp32-elf/\{runtime.tools.xtensa-esp32-elf-gcc.path\}/g' | \
+sed 's/{runtime\.platform\.path}.tools.xtensa-esp32s2-elf/\{runtime.tools.xtensa-esp32s2-elf-gcc.path\}/g' | \
+sed 's/{runtime\.platform\.path}.tools.xtensa-esp32s3-elf/\{runtime.tools.xtensa-esp32s3-elf-gcc.path\}/g' | \
+sed 's/{runtime\.platform\.path}.tools.riscv32-esp-elf-gdb/\{runtime.tools.riscv32-esp-elf-gdb.path\}/g' | \
+sed "s/{runtime\.platform\.path}.tools.riscv32-esp-elf/\\{runtime.tools.riscv32-esp-elf-gcc.path\\}/g" | \
+sed 's/{runtime\.platform\.path}.tools.esptool/\{runtime.tools.esptool_py.path\}/g' | \
+sed 's/{runtime\.platform\.path}.tools.openocd-esp32/\{runtime.tools.openocd-esp32.path\}/g' \
  > "$PKG_DIR/platform.txt"
 
+if ! [ -z ${VENDOR} ]; then
+    # Append vendor name to platform.txt to create a separate section
+    sed -i  "/^name=.*/s/$/ ($VENDOR)/" "$PKG_DIR/platform.txt"
+fi
+
 # Add header with version information
 echo "Generating core_version.h ..."
 ver_define=`echo $RELEASE_TAG | tr "[:lower:].\055" "[:upper:]_"`
@@ -301,15 +326,15 @@ if [ "$RELEASE_PRE" == "false" ]; then
     fi
 fi
 
-# Upload package JSONs
+# Upload package JSONs (temporary halted, until json generation is fixed)
 echo "Uploading $PACKAGE_JSON_DEV ..."
 echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
-echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
+# echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_DEV" "$OUTPUT_DIR/$PACKAGE_JSON_DEV"`
 echo
 if [ "$RELEASE_PRE" == "false" ]; then
     echo "Uploading $PACKAGE_JSON_REL ..."
     echo "Download URL: "`git_safe_upload_asset "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
-    echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
+    # echo "Pages URL: "`git_safe_upload_to_pages "$PACKAGE_JSON_REL" "$OUTPUT_DIR/$PACKAGE_JSON_REL"`
     echo
 fi
 

.github/scripts/sketch_utils.sh

@@ -27,6 +27,14 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
             shift
             sketchdir=$1
             ;;
+        -i )
+            shift
+            chunk_index=$1
+            ;;
+        -l )
+            shift
+            log_compilation=$1
+            ;;
         * )
             break
             ;;
@@ -127,6 +135,9 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
     fi
 
+    output_file="$HOME/.arduino/cli_compile_output.txt"
+    sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
+
     mkdir -p "$ARDUINO_CACHE_DIR"
     for i in `seq 0 $(($len - 1))`
     do
@@ -148,9 +159,43 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 --fqbn "$currfqbn" \
                 --board-options "$curroptions" \
                 --warnings "all" \
+                --build-property "compiler.warning_flags.all=-Wall -Werror=all -Wextra" \
                 --build-cache-path "$ARDUINO_CACHE_DIR" \
                 --build-path "$build_dir" \
-                $xtra_opts "${sketchdir}"
+                $xtra_opts "${sketchdir}" \
+                > $output_file
+
+            exit_status=$?
+            if [ $exit_status -ne 0 ]; then
+                echo ""ERROR: Compilation failed with error code $exit_status""
+                exit $exit_status
+            fi
+
+            if [ $log_compilation ]; then
+                #Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string
+                flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' $output_file | awk '{print $3}')
+                flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $5}' | tr -d '(%)')
+                ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' $output_file | awk '{print $4}')
+                ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $6}' | tr -d '(%)')
+
+                # Extract the directory path excluding the filename
+                directory_path=$(dirname "$sketch")
+                # Define the constant part
+                constant_part="/home/runner/Arduino/hardware/espressif/esp32/libraries/"
+                # Extract the desired substring using sed
+                lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
+                #append json file where key is fqbn, sketch name, sizes -> extracted values
+                echo "{\"name\": \"$lib_sketch_name\",
+                    \"sizes\": [{
+                            \"flash_bytes\": $flash_bytes,
+                            \"flash_percentage\": $flash_percentage,
+                            \"ram_bytes\": $ram_bytes,
+                            \"ram_percentage\": $ram_percentage
+                            }]
+                    }," >> "$sizes_file"
+            fi    
+
+
         elif [ -f "$ide_path/arduino-builder" ]; then
             echo "Building $sketchname with arduino-builder and FQBN=$currfqbn"
             echo "Build path = $build_dir"
@@ -252,6 +297,10 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             shift
             chunk_max=$1
             ;;
+        -l )
+            shift
+            log_compilation=$1
+            ;;
         * )
             break
             ;;
@@ -315,8 +364,19 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     echo "Start Sketch: $start_num"
     echo "End Sketch  : $end_index"
 
+    sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
+    if [ $log_compilation ]; then
+        #echo board,target and start of sketches to sizes_file json
+        echo "{ \"board\": \"$fqbn\",
+                \"target\": \"$target\",
+                \"sketches\": [" >> "$sizes_file"
+    fi
+
     local sketchnum=0
-    args+=" -ai $ide_path -au $user_path"
+    args+=" -ai $ide_path -au $user_path -i $chunk_index"
+    if [ $log_compilation ]; then
+        args+=" -l $log_compilation"
+    fi
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
@@ -333,6 +393,18 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             return $result
         fi
     done
+
+    if [ $log_compilation ]; then
+        #remove last comma from json
+        if [ $i -eq $(($len - 1)) ]; then
+            sed -i '$ s/.$//' "$sizes_file"
+        fi
+        #echo end of sketches sizes_file json
+        echo "]" >> "$sizes_file"
+        #echo end of board sizes_file json
+        echo "}," >> "$sizes_file"
+    fi
+
     return 0
 }
 

.github/workflows/lib.yml

@@ -109,12 +109,12 @@ jobs:
 
       - name: Push to github repo
         run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add ${{ env.RESULT_LIBRARY_TEST_FILE }}
           git commit -m "Generated External Libraries Test Results"
           git push origin HEAD:gh-pages
-          
+
   event_file:
     name: "Event File"
     if: |
@@ -126,4 +126,4 @@ jobs:
          uses: actions/upload-artifact@v2
          with:
            name: Event File
-           path: ${{github.event_path}}
+           path: ${{github.event_path}}