diff --git a/dvc/fs/ssh.py b/dvc/fs/ssh.py
index b5741cb569..9966d147ca 100644
--- a/dvc/fs/ssh.py
+++ b/dvc/fs/ssh.py
@@ -67,6 +67,11 @@ def _prepare_credentials(self, **config):
             or self.DEFAULT_PORT
         )
 
+        if config.get("ask_password") and config.get("password") is None:
+            config["password"] = ask_password(
+                login_info["host"], login_info["username"], login_info["port"]
+            )
+
         login_info["password"] = config.get("password")
         login_info["passphrase"] = config.get("password")
 
@@ -101,11 +106,6 @@ def _prepare_credentials(self, **config):
         login_info["agent_forwarding"] = config.get("agent_forwarding", True)
         login_info["proxy_command"] = user_ssh_config.get("ProxyCommand")
 
-        if config.get("ask_password") and login_info["password"] is None:
-            login_info["password"] = ask_password(
-                login_info["host"], login_info["username"], login_info["port"]
-            )
-
         # We are going to automatically add stuff to known_hosts
         # something like paramiko's AutoAddPolicy()
         login_info["known_hosts"] = None
diff --git a/tests/unit/fs/test_ssh.py b/tests/unit/fs/test_ssh.py
index a7372ddd65..60712e9b19 100644
--- a/tests/unit/fs/test_ssh.py
+++ b/tests/unit/fs/test_ssh.py
@@ -43,6 +43,12 @@ def test_init():
     assert fs.fs_args["password"] == fs.fs_args["passphrase"] == "xxx"
 
 
+def test_ssh_ask_password(mocker):
+    mocker.patch("dvc.fs.ssh.ask_password", return_value="fish")
+    fs = SSHFileSystem(user="test", host="2.2.2.2", ask_password=True)
+    assert fs.fs_args["password"] == fs.fs_args["passphrase"] == "fish"
+
+
 mock_ssh_config = """
 Host example.com
    User ubuntu