Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ensure BMC image is compatible with MB CPLD version
Summary: grandteton versions v2024.11.1 require MB CPLD version `20010` or greater. Added `checks_and_remediations` step to fail the upgrade if the MB CPLD version is incompatible with the image version.
Test Plan:
Unit tests, push flashy/bmc images to grandteton units e.g:
```
$ scripts/run_flashy_remote.sh --device mtd:flash1 --imagepath /tmp/tmp.H8gzWMVfip/openbmc.image.grandteton\:v2024.11.1/flash-grandteton --host twshared39604-oob.02.nao6.facebook.com --dry-run
```
Run step on a BMC with image version = v2024.11.1 and old MB CPLD (should fail):
```
# /run/flashy/flashy --install && /run/flashy/checks_and_remediations/grandteton/00_ensure_compatible_mb_cpld_version --imagepath /run/upgrade/image --device mtd:flash1
...
{"message":"S365492: Image version ('grandteton-v2024.11.1') requires MB CPLD version 20010 or greater (current MB CPLD version is 20006)"}
```
Run step on a BMC with image version = v2024.11.1 and new MB CPLD (should succeed):
```
/run/flashy/flashy --install && /run/flashy/checks_and_remediations/grandteton/00_ensure_compatible_mb_cpld_version --imagepath /run/upgrade/image --device mtd:flash1
...
2024/04/09 08:33:48 MB CPLD version 20010 is compatible with image version grandteton-v2024.11.1
```
Run step on a BMC with image version < v2024.11.1 and old MB CPLD (should succeed):
```
# /run/flashy/flashy --install && /run/flashy/checks_and_remediations/grandteton/00_ensure_compatible_mb_cpld_version --imagepath /run/upgrade/image --device mtd:flash1
...
2024/04/09 08:35:46 MB CPLD version 20006 is compatible with image version grandteton-v2023.39.2
```
Run step on a BMC with image version < v2024.11.1 and new MB CPLD (should succeed):
```
# /run/flashy/flashy --install && /run/flashy/checks_and_remediations/grandteton/00_ensure_compatible_mb_cpld_version --imagepath /run/upgrade/image --device mtd:flash1
...
2024/04/09 08:40:29 MB CPLD version 20010 is compatible with image version grandteton-v2023.39.2
```
Reviewed By: doranand
Differential Revision: D55921434
fbshipit-source-id: 8672d76c8f4ef18978cda8826f73395b1309b50c- Loading branch information
1 parent
ea2fc8c
commit 54734f8
Showing
5 changed files
with
238 additions
and
7 deletions.
There are no files selected for viewing
120 changes: 120 additions & 0 deletions
120
tools/flashy/checks_and_remediations/grandteton/00_ensure_compatible_mb_cpld_version.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /** | ||
| * Copyright 2020-present Facebook. All Rights Reserved. | ||
| * | ||
| * This program file is free software; you can redistribute it and/or modify it | ||
| * under the terms of the GNU General Public License as published by the | ||
| * Free Software Foundation; version 2 of the License. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, but WITHOUT | ||
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||
| * for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program in a file named COPYING; if not, write to the | ||
| * Free Software Foundation, Inc., | ||
| * 51 Franklin Street, Fifth Floor, | ||
| * Boston, MA 02110-1301 USA | ||
| */ | ||
|
|
||
| package remediations_grandteton | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "log" | ||
| "regexp" | ||
| "strconv" | ||
| "time" | ||
|
|
||
| "github.com/facebook/openbmc/tools/flashy/lib/step" | ||
| "github.com/facebook/openbmc/tools/flashy/lib/utils" | ||
| "github.com/facebook/openbmc/tools/flashy/lib/validate" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| func init() { | ||
| step.RegisterStep(ensureCompatibleMBCPLDVersion) | ||
| } | ||
|
|
||
| type FWUtilVersionJson struct { | ||
| COMPONENT string | ||
| FRU string | ||
| PRETTY_COMPONENT string | ||
| VERSION string | ||
| } | ||
|
|
||
| var versionRegex = regexp.MustCompile("^grandteton-(v[0-9]+[.][0-9]+[.][0-9]+)$") | ||
|
|
||
| var GetOpenBMCVersionFromImageFile = validate.GetOpenBMCVersionFromImageFile | ||
|
|
||
| // ensureCompatibleMBCPLDVersion checks whether the MB CPLD version is compatible with the image being | ||
| // flashed | ||
| func ensureCompatibleMBCPLDVersion(stepParams step.StepParams) step.StepExitError { | ||
| // Fetch current MB CPLD version from fw-util | ||
| mbCpldVersion, stepErr := fwutilGetMbCpldVersion() | ||
| if stepErr != nil { | ||
| return stepErr | ||
| } | ||
|
|
||
| // Fetch BMC version from image file | ||
| imageFileVer, err := GetOpenBMCVersionFromImageFile(stepParams.ImageFilePath) | ||
| if err != nil { | ||
| return step.ExitSafeToReboot{ | ||
| Err: err, | ||
| } | ||
| } | ||
|
|
||
| if !versionRegex.MatchString(imageFileVer) { | ||
| return step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("Image version '%v' does not match expected format %v", imageFileVer, versionRegex), | ||
| } | ||
| } | ||
|
|
||
| // S365492: Images newer than v2024.11.1 require MB CPLD version 20010 or greater, otherwise sensor data | ||
| // becomes unavailable (T175831581). Booting older images with the new CPLD version should be safe. | ||
| if imageFileVer >= "grandteton-v2024.11.1" && mbCpldVersion < 20010 { | ||
| return step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("S365492: Image version ('%v') requires MB CPLD version 20010 or greater (current MB CPLD version is %d)", imageFileVer, mbCpldVersion), | ||
| } | ||
| } | ||
|
|
||
| log.Printf("MB CPLD version %d is compatible with image version %v", mbCpldVersion, imageFileVer) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func fwutilGetMbCpldVersion() (int, step.StepExitError) { | ||
| // Run fw-util | ||
| fwutilCmd := []string{ | ||
| "fw-util", "mb", "--version-json", "mb_cpld", | ||
| } | ||
| exitCode, err, stdout, stderr := utils.RunCommand(fwutilCmd, 1800*time.Second) | ||
|
|
||
| if err != nil { | ||
| return 0, step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("`fw-util mb --version-json mb_cpld` failed with exit code %d, stderr: %v, stdout: %v", exitCode, stderr, stdout), | ||
| } | ||
| } | ||
|
|
||
| var fwutilVersionJson []FWUtilVersionJson | ||
| if err := json.Unmarshal([]byte(stdout), &fwutilVersionJson); err != nil { | ||
| return 0, step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("failed to parse `fw-util mb --version-json mb_cpld` output: %v", err), | ||
| } | ||
| } | ||
|
|
||
| if len(fwutilVersionJson) != 1 { | ||
| return 0, step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("`fw-util mb --version-json mb_cpld` returned %v entries", len(fwutilVersionJson)), | ||
| } | ||
| } | ||
|
|
||
| mbCpldVersion, err := strconv.Atoi(fwutilVersionJson[0].VERSION) | ||
| if err != nil { | ||
| return 0, step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("Cannot parse `fw-util mb --version-json mb_cpld` version as integer: %v", fwutilVersionJson), | ||
| } | ||
| } | ||
|
|
||
| return mbCpldVersion, nil | ||
| } |
110 changes: 110 additions & 0 deletions
110
tools/flashy/checks_and_remediations/grandteton/00_ensure_compatible_mb_cpld_version_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| package remediations_grandteton | ||
|
|
||
| import ( | ||
| "strings" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/facebook/openbmc/tools/flashy/lib/step" | ||
| "github.com/facebook/openbmc/tools/flashy/lib/utils" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| var fwUtilOutputOldCpldExample = ` | ||
| [ | ||
| { | ||
| "COMPONENT": "mb_cpld", | ||
| "FRU": "mb", | ||
| "PRETTY_COMPONENT": "MB_CPLD", | ||
| "VERSION": "00020008" | ||
| } | ||
| ] | ||
| ` | ||
|
|
||
| var fwUtilOutputNewCpldExample = ` | ||
| [ | ||
| { | ||
| "COMPONENT": "mb_cpld", | ||
| "FRU": "mb", | ||
| "PRETTY_COMPONENT": "MB_CPLD", | ||
| "VERSION": "00020010" | ||
| } | ||
| ] | ||
| ` | ||
|
|
||
| func TestEnsureCompatibleMBCPLDVersion(t *testing.T) { | ||
| // mock and defer restore RunCommand | ||
| runCommandOrig := utils.RunCommand | ||
| getOpenBMCVersionFromImageFileOrig := GetOpenBMCVersionFromImageFile | ||
| defer func() { | ||
| utils.RunCommand = runCommandOrig | ||
| GetOpenBMCVersionFromImageFile = getOpenBMCVersionFromImageFileOrig | ||
| }() | ||
| cases := []struct { | ||
| name string | ||
| bmcImageVersion string | ||
| runCmdStdout string | ||
| want step.StepExitError | ||
| }{ | ||
| { | ||
| name: "succeeded as image version == v2023.39.2 and CPLD is old", | ||
| bmcImageVersion: "grandteton-v2023.39.2", | ||
| runCmdStdout: fwUtilOutputOldCpldExample, | ||
| want: nil, | ||
| }, | ||
| { | ||
| name: "succeeded as image version == v2024.11.1 and CPLD is new", | ||
| bmcImageVersion: "grandteton-v2024.11.1", | ||
| runCmdStdout: fwUtilOutputNewCpldExample, | ||
| want: nil, | ||
| }, | ||
| { | ||
| name: "succeeded as image version > v2024.11.1 and CPLD is new", | ||
| bmcImageVersion: "grandteton-v2024.12.1", | ||
| runCmdStdout: fwUtilOutputNewCpldExample, | ||
| want: nil, | ||
| }, | ||
| { | ||
| name: "failed as image version == v2024.11.1 but CPLD is old", | ||
| bmcImageVersion: "grandteton-v2024.11.1", | ||
| runCmdStdout: fwUtilOutputOldCpldExample, | ||
| want: step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("S365492: Image version ('grandteton-v2024.11.1') requires MB CPLD version 20010 or greater (current MB CPLD version is 20008)"), | ||
| }, | ||
| }, | ||
| { | ||
| name: "failed as image version > v2024.11.1 but CPLD is old", | ||
| bmcImageVersion: "grandteton-v2024.12.0", | ||
| runCmdStdout: fwUtilOutputOldCpldExample, | ||
| want: step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("S365492: Image version ('grandteton-v2024.12.0') requires MB CPLD version 20010 or greater (current MB CPLD version is 20008)"), | ||
| }, | ||
| }, | ||
| { | ||
| name: "invalid image version", | ||
| bmcImageVersion: "fby3-v2024.12.0", | ||
| runCmdStdout: fwUtilOutputOldCpldExample, | ||
| want: step.ExitSafeToReboot{ | ||
| Err: errors.Errorf("Image version 'fby3-v2024.12.0' does not match expected format ^grandteton-(v[0-9]+[.][0-9]+[.][0-9]+)$"), | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| wantCmd := "fw-util mb --version-json mb_cpld" | ||
| for _, tc := range cases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| utils.RunCommand = func(cmdArr []string, timeout time.Duration) (int, error, string, string) { | ||
| gotCmd := strings.Join(cmdArr, " ") | ||
| if wantCmd != gotCmd { | ||
| t.Errorf("cmd: want '%v' got '%v'", wantCmd, gotCmd) | ||
| } | ||
| return 0, nil, tc.runCmdStdout, "" | ||
| } | ||
| GetOpenBMCVersionFromImageFile = func(imageFile string) (string, error) { | ||
| return tc.bmcImageVersion, nil | ||
| } | ||
| got := ensureCompatibleMBCPLDVersion(step.StepParams{}) | ||
| step.CompareTestExitErrors(tc.want, got, t) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters