Merge branch 'master' into idf-release/v5.1

.flake8

@@ -1,12 +1,10 @@
 # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-python/.flake8
 # See: https://flake8.pycqa.org/en/latest/user/configuration.html
-# The code style defined in this file is the official standardized style to be used in all Arduino tooling projects and
-# should not be modified.
 
 [flake8]
 doctests = True
 # W503 and W504 are mutually exclusive. PEP 8 recommends line break before.
 ignore = W503,E203
-max-complexity = 10
+max-complexity = 20
 max-line-length = 120
 select = E,W,F,C,N

.github/scripts/sketch_utils.sh

@@ -121,7 +121,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
     fi
 
     if [ -z "$fqbn" ]; then
-        echo "No FQBN passed or unvalid chip: $target"
+        echo "No FQBN passed or invalid chip: $target"
         exit 1
     fi
 
@@ -139,7 +139,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         echo "Skipping $sketchname for target $target"
         exit 0
     fi
-    
+
     ARDUINO_CACHE_DIR="$HOME/.arduino/cache.tmp"
     if [ -n "$ARDUINO_BUILD_DIR" ]; then
         build_dir="$ARDUINO_BUILD_DIR"
@@ -177,7 +177,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 --build-path "$build_dir" \
                 $xtra_opts "${sketchdir}" \
                 > $output_file
-            
+
             exit_status=$?
             if [ $exit_status -ne 0 ]; then
                 echo ""ERROR: Compilation failed with error code $exit_status""
@@ -198,11 +198,11 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 # Extract the desired substring using sed
                 lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
                 #append json file where key is fqbn, sketch name, sizes -> extracted values
-                echo "{\"name\": \"$lib_sketch_name\", 
+                echo "{\"name\": \"$lib_sketch_name\",
                     \"sizes\": [{
-                            \"flash_bytes\": $flash_bytes, 
-                            \"flash_percentage\": $flash_percentage, 
-                            \"ram_bytes\": $ram_bytes, 
+                            \"flash_bytes\": $flash_bytes,
+                            \"flash_percentage\": $flash_percentage,
+                            \"ram_bytes\": $ram_bytes,
                             \"ram_percentage\": $ram_percentage
                             }]
                     }," >> "$sizes_file"
@@ -365,6 +365,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
         start_index=$(( $chunk_index * $chunk_size ))
         if [ "$sketchcount" -le "$start_index" ]; then
             echo "Skipping job"
+            touch ~/.build_skipped
             return 0
         fi
 
@@ -386,7 +387,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     if [ $log_compilation ]; then
         #echo board,target and start of sketches to sizes_file json
         echo "{ \"board\": \"$fqbn\",
-                \"target\": \"$target\", 
+                \"target\": \"$target\",
                 \"sketches\": [" >> "$sizes_file"
     fi
 

.github/scripts/tests_build.sh

@@ -2,19 +2,20 @@
 
 USAGE="
 USAGE:
-    ${0} -c <chunk_build_opts>
-       Example: ${0} -c -t esp32 -i 0 -m 15
+    ${0} -c -type <test_type> <chunk_build_opts>
+       Example: ${0} -c -type validation -t esp32 -i 0 -m 15
     ${0} -s sketch_name <build_opts>
        Example: ${0} -s hello_world -t esp32
     ${0} -clean
        Remove build and test generated files
 "
 
 function clean(){
-    rm -rf tests/*/build*/
     rm -rf tests/.pytest_cache
-    rm -rf tests/*/__pycache__/
-    rm -rf tests/*/*.xml
+    find tests/ -type d -name 'build*' -exec rm -rf "{}" \+
+    find tests/ -type d -name '__pycache__' -exec rm -rf "{}" \+
+    find tests/ -name '*.xml' -exec rm -rf "{}" \+
+    find tests/ -name 'result_*.json' -exec rm -rf "{}" \+
 }
 
 SCRIPTS_DIR="./.github/scripts"
@@ -35,6 +36,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     -clean )
         clean
         exit 0
@@ -52,12 +57,25 @@ source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh
 
 args="-ai $ARDUINO_IDE_PATH -au $ARDUINO_USR_PATH"
 
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_build -eq 1 ]; then
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
-    args+=" -p $PWD/tests"
+    args+=" -p $test_folder"
 else
     BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
-    args+=" -s $PWD/tests/$sketch"
+    args+=" -s $test_folder/$sketch"
 fi
 
 ${BUILD_CMD} ${args} $*

.github/scripts/tests_run.sh

@@ -8,16 +8,21 @@ function run_test() {
     local sketchdir=$(dirname $sketch)
     local sketchname=$(basename $sketchdir)
 
+    if [[ -f "$sketchdir/.skip.$platform" ]]; then
+      echo "Skipping $sketchname test in $target"
+      exit 0
+    fi
+
     if [ $options -eq 0 ] && [ -f $sketchdir/cfg.json ]; then
         len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json`
     else
         len=1
     fi
 
     if [ $len -eq 1 ]; then
-      # build_dir="tests/$sketchname/build"
+      # build_dir="$sketchdir/build"
       build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
-      report_file="tests/$sketchname/$sketchname.xml"
+      report_file="$sketchdir/$sketchname.xml"
     fi
 
     for i in `seq 0 $(($len - 1))`
@@ -28,12 +33,21 @@ function run_test() {
         fi
 
         if [ $len -ne 1 ]; then
-            # build_dir="tests/$sketchname/build$i"
+            # build_dir="$sketchdir/build$i"
             build_dir="$HOME/.arduino/tests/$sketchname/build$i.tmp"
-            report_file="tests/$sketchname/$sketchname$i.xml"
+            report_file="$sketchdir/$sketchname$i.xml"
+        fi
+
+        if [ $platform == "wokwi" ]; then
+            extra_args="--target $target --embedded-services arduino,wokwi --wokwi-timeout=$wokwi_timeout"
+            if [[ -f "$sketchdir/scenario.yaml" ]]; then
+                extra_args+=" --wokwi-scenario $sketchdir/scenario.yaml"
+            fi
+        else
+            extra_args="--embedded-services esp,arduino"
         fi
 
-        pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file
+        pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args
         result=$?
         if [ $result -ne 0 ]; then
             return $result
@@ -44,6 +58,8 @@ function run_test() {
 SCRIPTS_DIR="./.github/scripts"
 COUNT_SKETCHES="${SCRIPTS_DIR}/sketch_utils.sh count"
 
+platform="hardware"
+wokwi_timeout=60000
 chunk_run=0
 options=0
 erase=0
@@ -53,6 +69,11 @@ while [ ! -z "$1" ]; do
     -c )
         chunk_run=1
         ;;
+    -w )
+        shift
+        wokwi_timeout=$1
+        platform="wokwi"
+        ;;
     -o )
         options=1
         ;;
@@ -79,6 +100,10 @@ while [ ! -z "$1" ]; do
         echo "$USAGE"
         exit 0
         ;;
+    -type )
+        shift
+        test_type=$1
+        ;;
     * )
       break
       ;;
@@ -88,21 +113,39 @@ done
 
 source ${SCRIPTS_DIR}/install-arduino-ide.sh
 
+# If sketch is provided and test type is not, test type is inferred from the sketch path
+if [[ $test_type == "all" ]] || [[ -z $test_type ]]; then
+    if [ -n "$sketch" ]; then
+        tmp_sketch_path=$(find tests -name $sketch.ino)
+        test_type=$(basename $(dirname $(dirname "$tmp_sketch_path")))
+        echo "Sketch $sketch test type: $test_type"
+        test_folder="$PWD/tests/$test_type"
+    else
+      test_folder="$PWD/tests"
+    fi
+else
+    test_folder="$PWD/tests/$test_type"
+fi
+
 if [ $chunk_run -eq 0 ]; then
-    run_test $target $PWD/tests/$sketch/$sketch.ino $options $erase
+    if [ -z $sketch ]; then
+        echo "ERROR: Sketch name is required for single test run"
+        exit 1
+    fi
+    run_test $target $test_folder/$sketch/$sketch.ino $options $erase
 else
   if [ "$chunk_max" -le 0 ]; then
       echo "ERROR: Chunks count must be positive number"
-      return 1
+      exit 1
   fi
 
   if [ "$chunk_index" -ge "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
       echo "ERROR: Chunk index must be less than chunks count"
-      return 1
+      exit 1
   fi
 
   set +e
-  ${COUNT_SKETCHES} $PWD/tests $target
+  ${COUNT_SKETCHES} $test_folder $target
   sketchcount=$?
   set -e
   sketches=$(cat sketches.txt)
@@ -123,7 +166,8 @@ else
       start_index=$(( $chunk_index * $chunk_size ))
       if [ "$sketchcount" -le "$start_index" ]; then
           echo "Skipping job"
-          return 0
+          touch ~/.test_skipped
+          exit 0
       fi
 
       end_index=$(( $(( $chunk_index + 1 )) * $chunk_size ))