ELBERT: Harden bios_util.sh (#210) (#120)

Summary:
ELBERT: bios_util.sh: Improve stability

- Add retry_command method to board-utils.sh. This might later be expanded to other utilities.
- Split bios program into write and verify stages, where the verify stage is retried to remove false negatives

Testing:
```
With up to 5 retries, 340 program cycles of alternating images succeeded.
The false positive failure rate is about 15% for the full 16MB image. The reason for these false readings is being investigated/
With 5 retries, the theoretical failure rate matched the measured order for subsequent retries.

e.g.:

Theoretical:
1) 0.156^0 * 0.844  = 84.4%
2) 0.156^1 * 0.844  = 13.16%
3) 0.156^2 * 0.844  = 2.05%
4) 0.156^3 * 0.844  = 0.32%
5) 0.156^4 * 0.844  = 0.05%

Measured: Out of 346 passed programming cycles: The following indicated how many read attempts it took to match the expected image.

# Attempt 1 84.40%
292
# Attempt 2 13.00%
45
# Attempt 3 1.73%
6
# Attempt 4 0.87%
3
# Attempt 5 0%
0

Pull Request resolved: facebookexternal/openbmc.arista#120

Reviewed By: mikechoifb

fbshipit-source-id: c365e5d741

meta-facebook/meta-elbert/recipes-utils/openbmc-utils/files/bios_util.sh

@@ -10,7 +10,7 @@ usage() {
     program=$(basename "$0")
     echo "Usage:"
     echo "$program <OP> <bios file>"
-    echo "      <OP> : read, write, erase, recover"
+    echo "      <OP> : read, write, erase, verify"
     exit 1
 }
 
@@ -44,13 +44,21 @@ fi
 
 if [ "$1" = "erase" ]; then
     echo "Erasing flash content ..."
-    flashrom -p linux_spi:dev=/dev/spidev1.0 -E -c $CHIPTYPE
+    flashrom -f -p linux_spi:dev=/dev/spidev1.0 -E -c $CHIPTYPE || exit 1
 elif [ "$1" = "read" ]; then
     echo "Reading flash content..."
-    flashrom -p linux_spi:dev=/dev/spidev1.0 -r "$2" -c $CHIPTYPE
+    flashrom -p linux_spi:dev=/dev/spidev1.0 -r "$2" -c $CHIPTYPE || exit 1
+elif [ "$1" = "verify" ]; then
+    echo "Verifying flash content..."
+    flashrom -f -p linux_spi:dev=/dev/spidev1.0 -v "$2" -c $CHIPTYPE || exit 1
 elif [ "$1" = "write" ]; then
     echo "Writing flash content..."
-    flashrom -p linux_spi:dev=/dev/spidev1.0 -w "$2" -c $CHIPTYPE || exit 1
+    flashrom -n -f -p linux_spi:dev=/dev/spidev1.0 -w "$2" -c $CHIPTYPE || exit 1
+    echo "Verifying flash content..."
+    # ELBERTTODO understand why read is flaky, giving false verification fails
+    # Retry verification up to 5 times
+    retry_command 5 flashrom -f -p linux_spi:dev=/dev/spidev1.0 \
+       -v "$2" -c $CHIPTYPE || exit 1
 else
     usage
 fi

meta-facebook/meta-elbert/recipes-utils/openbmc-utils/files/board-utils.sh

@@ -212,3 +212,22 @@ power_off_pim() {
     fi
     logger pim_enable: powered off PIM"${pim}"
 }
+
+retry_command() {
+    # Retry command up to $1 attempts
+    local retries=$1
+    shift
+
+    local count=0
+    until "$@"; do
+        exit=$?
+        count=$((count+1))
+        if [ "$count" -lt "$retries" ]; then
+            echo "Attempt $count/$retries failed with $exit, retrying..."
+        else
+            echo "Retry $count/$retries failed with $exit, no more retries left"
+            return $exit
+        fi
+    done
+    return 0
+}