diff --git a/lib/puppet_x/certs/provider/keystore.rb b/lib/puppet_x/certs/provider/keystore.rb
index 0a14234c..d91eab3d 100644
--- a/lib/puppet_x/certs/provider/keystore.rb
+++ b/lib/puppet_x/certs/provider/keystore.rb
@@ -20,7 +20,7 @@ def exists?
               '-storepass:file', resource[:password_file],
             )
           rescue Puppet::ExecutionFailure => e
-            if e.message.include?('java.security.UnrecoverableKeyException')
+            if e.message.include?('java.security.UnrecoverableKeyException') || e.message.include?('keystore password was incorrect')
               Puppet.debug("Invalid password for #{store}")
               return false
             else
diff --git a/spec/acceptance/truststore_spec.rb b/spec/acceptance/truststore_spec.rb
index 6c32dafc..845fb323 100644
--- a/spec/acceptance/truststore_spec.rb
+++ b/spec/acceptance/truststore_spec.rb
@@ -41,7 +41,7 @@
       it { should be_grouped_into 'root' }
     end
 
-    describe command("keytool -list -keystore #{truststore_path} -storepass:file #{truststore_password_file}") do
+    describe command("keytool -list -keystore #{truststore_path} -storepass testpassword") do
       its(:exit_status) { should eq 0 }
       its(:stdout) { should match(/^Keystore type: PKCS12$/i) }
       its(:stdout) { should match(/^Your keystore contains 0 entries$/) }
@@ -87,5 +87,115 @@
       its(:stdout) { should match(/^Owner: CN=#{host_inventory['fqdn']}$/) }
       its(:stdout) { should match(/^Issuer: CN=#{host_inventory['fqdn']}$/) }
     end
+
+    describe 'changing password' do
+      describe 'apply puppet' do
+        let(:manifest) do
+          <<-PUPPET
+          $truststore_password_file = '/etc/pki/truststore_password-file'
+
+          package { 'java-11-openjdk-headless':
+            ensure => installed,
+          }
+
+          file { $truststore_password_file:
+            ensure    => file,
+            content   => 'other-password',
+            owner     => 'root',
+            group     => 'root',
+            mode      => '0440',
+            show_diff => false,
+          }
+
+          truststore { "/etc/pki/truststore":
+            ensure        => present,
+            password_file => $truststore_password_file,
+            owner         => 'root',
+            group         => 'root',
+            mode          => '0640',
+          }
+          PUPPET
+        end
+
+        it 'applies changes with no errors' do
+          apply_manifest_on(default, manifest, expect_changes: true)
+        end
+
+        it 'applies a second time without changes' do
+          apply_manifest_on(default, manifest, catch_changes: true)
+        end
+      end
+
+      describe command("keytool -list -keystore #{truststore_path} -storepass other-password") do
+        its(:exit_status) { should eq 0 }
+        its(:stdout) { should match(/^Keystore type: PKCS12$/i) }
+        its(:stdout) { should match(/^Your keystore contains 0 entries$/) }
+      end
+    end
+
+    describe 'noop' do
+      describe 'change password file' do
+        let(:manifest) do
+          <<-PUPPET
+          file { '/etc/pki/truststore_password-file':
+            ensure    => file,
+            content   => 'wrong-password',
+            owner     => 'root',
+            group     => 'root',
+            mode      => '0440',
+            show_diff => false,
+          }
+          PUPPET
+        end
+
+        it 'applies changes with no errors' do
+          apply_manifest_on(default, manifest, catch_failures: true)
+        end
+      end
+
+      describe 'run in noop mode with wrong password' do
+        let(:manifest) do
+          <<-PUPPET
+          $truststore_password_file = '/etc/pki/truststore_password-file'
+
+          package { 'java-11-openjdk-headless':
+            ensure => installed,
+          }
+
+          file { $truststore_password_file:
+            ensure    => file,
+            content   => 'other-password',
+            owner     => 'root',
+            group     => 'root',
+            mode      => '0440',
+            show_diff => false,
+          }
+
+          truststore { "/etc/pki/truststore":
+            ensure        => present,
+            password_file => $truststore_password_file,
+            owner         => 'root',
+            group         => 'root',
+            mode          => '0640',
+          }
+          PUPPET
+        end
+
+        it 'applies changes with no errors' do
+          apply_manifest_on(default, manifest, noop: true)
+        end
+      end
+
+      describe file(truststore_path) do
+        it { is_expected.to be_file }
+      end
+
+      # Should still be readable with the old password
+      describe command("keytool -list -keystore #{truststore_path} -storepass other-password") do
+        its(:exit_status) { should eq 0 }
+        its(:stdout) { should match(/^Keystore type: PKCS12$/i) }
+        its(:stdout) { should match(/^Your keystore contains 0 entries$/) }
+      end
+    end
   end
 end