diff --git a/commands/rm.go b/commands/rm.go
index f14c1d2500c6..44151121bcc6 100644
--- a/commands/rm.go
+++ b/commands/rm.go
@@ -74,15 +74,15 @@ func runRm(ctx context.Context, dockerCli command.Cli, in rmOptions) error {
 					return err
 				}
 
+				if b.DockerContext {
+					return errors.Errorf("context builder cannot be removed, run `docker context rm %s` to remove this context", b.Name)
+				}
+
 				nodes, err := b.LoadNodes(timeoutCtx)
 				if err != nil {
 					return err
 				}
 
-				if cb := b.ContextName(); cb != "" {
-					return errors.Errorf("context builder cannot be removed, run `docker context rm %s` to remove this context", cb)
-				}
-
 				err1 := rm(ctx, nodes, in)
 				if err := txn.Remove(b.Name); err != nil {
 					return err