diff --git a/CHANGES/6291.feature b/CHANGES/6291.feature
new file mode 100644
index 00000000000..4be34b0ecfb
--- /dev/null
+++ b/CHANGES/6291.feature
@@ -0,0 +1 @@
+Added warning in SigningService for signing script if it has changed on disk
diff --git a/pulpcore/app/migrations/0024_signingservice_sha256.py b/pulpcore/app/migrations/0024_signingservice_sha256.py
new file mode 100644
index 00000000000..3fd67d51d4b
--- /dev/null
+++ b/pulpcore/app/migrations/0024_signingservice_sha256.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2.11 on 2020-04-27 07:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0023_change_exporter_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='signingservice',
+            name='sha256',
+            field=models.CharField(max_length=64, null=True),
+        ),
+    ]
diff --git a/pulpcore/app/models/content.py b/pulpcore/app/models/content.py
index 6dc961cc016..04552382f11 100644
--- a/pulpcore/app/models/content.py
+++ b/pulpcore/app/models/content.py
@@ -5,6 +5,7 @@
 import hashlib
 import tempfile
 import subprocess
+import warnings
 
 import gnupg
 
@@ -387,6 +388,7 @@ class SigningService(BaseModel):
     """
     name = models.TextField(db_index=True, unique=True)
     script = models.TextField()
+    sha256 = models.CharField(max_length=64, null=True)
 
     def sign(self, filename):
         """
@@ -412,6 +414,9 @@ def sign(self, filename):
             stderr=subprocess.PIPE,
         )
 
+        if self.sha256 != self.hash_value(self.script):
+            warnings.warn('Provided signing script does not match original signing script', Warning)
+
         if completed_process.returncode != 0:
             raise RuntimeError(str(completed_process.stderr))
 
@@ -424,7 +429,7 @@ def sign(self, filename):
 
     def validate(self):
         """
-        Ensure that the external signing script produces the desired beahviour.
+        Ensure that the external signing script produces the desired behaviour.
 
         With desired behaviour we mean the behaviour as validated by this method. Subclasses are
         required to implement this method. Works by calling the sign() method on some test data, and
@@ -440,9 +445,20 @@ def save(self, *args, **kwargs):
         """
         Save a signing service to the database (unless it fails to validate).
         """
+        self.sha256 = self.hash_value(self.script)
         self.validate()
         super().save(*args, **kwargs)
 
+    def hash_value(self, filename):
+        """
+        Calculate hash value (sha256) of signing script.
+        """
+        sha256_hash = hashlib.sha256()
+        with open(filename, "rb") as f:
+            for byte_block in iter(lambda: f.read(4096), b""):
+                sha256_hash.update(byte_block)
+            return sha256_hash.hexdigest()
+
 
 class AsciiArmoredDetachedSigningService(SigningService):
     """