diff --git a/CHANGELOG.md b/CHANGELOG.md
index e181fe9b0a79..5e678450c462 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### Bug Fixes
 
+* (cli) [#12127](https://github.com/cosmos/cosmos-sdk/pull/12127) Fix the CLI not always taking into account `--fee-payer` and `--fee-granter` flags.
 * (migrations) [#12028](https://github.com/cosmos/cosmos-sdk/pull/12028) Fix v0.45->v0.46 in-place store migrations.
 * (baseapp) [#12089](https://github.com/cosmos/cosmos-sdk/pull/12089) Include antehandler and runMsgs events in SimulateTx.
 * (cli) [#12095](https://github.com/cosmos/cosmos-sdk/pull/12095) Fix running a tx with --dry-run returns an error
diff --git a/client/tx/factory.go b/client/tx/factory.go
index ab6ab7943793..47ad7febb08d 100644
--- a/client/tx/factory.go
+++ b/client/tx/factory.go
@@ -34,6 +34,8 @@ type Factory struct {
 	memo               string
 	fees               sdk.Coins
 	tip                *tx.Tip
+	feeGranter         sdk.AccAddress
+	feePayer           sdk.AccAddress
 	gasPrices          sdk.DecCoins
 	signMode           signing.SignMode
 	simulateAndExecute bool
@@ -79,6 +81,8 @@ func NewFactoryCLI(clientCtx client.Context, flagSet *pflag.FlagSet) Factory {
 		gasAdjustment:      gasAdj,
 		memo:               memo,
 		signMode:           signMode,
+		feeGranter:         clientCtx.FeeGranter,
+		feePayer:           clientCtx.FeePayer,
 	}
 
 	feesStr, _ := flagSet.GetString(flags.FlagFees)
@@ -225,6 +229,18 @@ func (f Factory) WithTimeoutHeight(height uint64) Factory {
 	return f
 }
 
+// WithFeeGranter returns a copy of the Factory with an updated fee granter.
+func (f Factory) WithFeeGranter(fg sdk.AccAddress) Factory {
+	f.feeGranter = fg
+	return f
+}
+
+// WithFeePayer returns a copy of the Factory with an updated fee granter.
+func (f Factory) WithFeePayer(fp sdk.AccAddress) Factory {
+	f.feePayer = fp
+	return f
+}
+
 // BuildUnsignedTx builds a transaction to be signed given a set of messages.
 // Once created, the fee, memo, and messages are set.
 func (f Factory) BuildUnsignedTx(msgs ...sdk.Msg) (client.TxBuilder, error) {
@@ -264,6 +280,8 @@ func (f Factory) BuildUnsignedTx(msgs ...sdk.Msg) (client.TxBuilder, error) {
 	tx.SetMemo(f.memo)
 	tx.SetFeeAmount(fees)
 	tx.SetGasLimit(f.gas)
+	tx.SetFeeGranter(f.feeGranter)
+	tx.SetFeePayer(f.feePayer)
 	tx.SetTimeoutHeight(f.TimeoutHeight())
 
 	return tx, nil
diff --git a/client/tx/tx.go b/client/tx/tx.go
index 4605d4991ce4..92eb733ed08e 100644
--- a/client/tx/tx.go
+++ b/client/tx/tx.go
@@ -106,8 +106,6 @@ func BroadcastTx(clientCtx client.Context, txf Factory, msgs ...sdk.Msg) error {
 		}
 	}
 
-	tx.SetFeeGranter(clientCtx.GetFeeGranterAddress())
-	tx.SetFeePayer(clientCtx.GetFeePayerAddress())
 	err = Sign(txf, clientCtx.GetFromName(), tx, true)
 	if err != nil {
 		return err
diff --git a/x/feegrant/client/testutil/suite.go b/x/feegrant/client/testutil/suite.go
index d3bd39e313b7..06042e9b5f7c 100644
--- a/x/feegrant/client/testutil/suite.go
+++ b/x/feegrant/client/testutil/suite.go
@@ -750,17 +750,55 @@ func (s *IntegrationTestSuite) TestTxWithFeeGrant() {
 	_, err = s.network.WaitForHeight(1)
 	s.Require().NoError(err)
 
-	// granted fee allowance for an account which is not in state and creating
-	// any tx with it by using --fee-account shouldn't fail
-	out, err := govtestutil.MsgSubmitLegacyProposal(val.ClientCtx, grantee.String(),
-		"Text Proposal", "No desc", govv1beta1.ProposalTypeText,
-		fmt.Sprintf("--%s=%s", flags.FlagFeeGranter, granter.String()),
-	)
+	testcases := []struct {
+		name       string
+		from       string
+		flags      []string
+		expErrCode uint32
+	}{
+		{
+			name:  "granted fee allowance for an account which is not in state and creating any tx with it by using --fee-granter shouldn't fail",
+			from:  grantee.String(),
+			flags: []string{fmt.Sprintf("--%s=%s", flags.FlagFeeGranter, granter.String())},
+		},
+		{
+			name:       "--fee-payer should also sign the tx (direct)",
+			from:       grantee.String(),
+			flags:      []string{fmt.Sprintf("--%s=%s", flags.FlagFeePayer, granter.String())},
+			expErrCode: 4,
+		},
+		{
+			name: "--fee-payer should also sign the tx (amino-json)",
+			from: grantee.String(),
+			flags: []string{
+				fmt.Sprintf("--%s=%s", flags.FlagFeePayer, granter.String()),
+				fmt.Sprintf("--%s=%s", flags.FlagSignMode, flags.SignModeLegacyAminoJSON),
+			},
+			expErrCode: 4,
+		},
+		{
+			name: "use --fee-payer and --fee-granter together works",
+			from: grantee.String(),
+			flags: []string{
+				fmt.Sprintf("--%s=%s", flags.FlagFeePayer, grantee.String()),
+				fmt.Sprintf("--%s=%s", flags.FlagFeeGranter, granter.String())},
+		},
+	}
+
+	for _, tc := range testcases {
+		s.Run(tc.name, func() {
+			out, err := govtestutil.MsgSubmitLegacyProposal(val.ClientCtx, tc.from,
+				"Text Proposal", "No desc", govv1beta1.ProposalTypeText,
+				tc.flags...,
+			)
+			s.Require().NoError(err)
+
+			var resp sdk.TxResponse
+			s.Require().NoError(clientCtx.Codec.UnmarshalJSON(out.Bytes(), &resp), out.String())
+			s.Require().Equal(tc.expErrCode, resp.Code, resp)
+		})
+	}
 
-	s.Require().NoError(err)
-	var resp sdk.TxResponse
-	s.Require().NoError(clientCtx.Codec.UnmarshalJSON(out.Bytes(), &resp), out.String())
-	s.Require().Equal(uint32(0), resp.Code)
 }
 
 func (s *IntegrationTestSuite) TestFilteredFeeAllowance() {