diff --git a/src/main/kotlin/org/jitsi/nlj/srtp/SrtpConfig.kt b/src/main/kotlin/org/jitsi/nlj/srtp/SrtpConfig.kt
new file mode 100644
index 000000000..0c15dbea5
--- /dev/null
+++ b/src/main/kotlin/org/jitsi/nlj/srtp/SrtpConfig.kt
@@ -0,0 +1,27 @@
+/*
+ * Copyright @ 2019 - present 8x8, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jitsi.nlj.srtp
+
+import org.jitsi.config.JitsiConfig
+import org.jitsi.metaconfig.config
+
+class SrtpConfig {
+    companion object {
+        val maxConsecutivePacketsDiscardedEarly: Int by config {
+            "jmt.srtp.max-consecutive-packets-discarded-early".from(JitsiConfig.newConfig)
+        }
+    }
+}
diff --git a/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt b/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt
index af0bddf3c..f51773639 100644
--- a/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt
+++ b/src/main/kotlin/org/jitsi/nlj/srtp/SrtpTransformer.kt
@@ -29,6 +29,7 @@ import org.jitsi.utils.logging2.Logger
 import org.jitsi.utils.logging2.createChildLogger
 import org.jitsi.utils.logging2.cwarn
 import java.util.concurrent.ConcurrentHashMap
+import org.jitsi.nlj.srtp.SrtpConfig.Companion.maxConsecutivePacketsDiscardedEarly
 
 /**
  * Implements the methods common to all 4 transformer implementation (encrypt/decrypt for SRTP/SRTCP)
@@ -177,13 +178,34 @@ class SrtpDecryptTransformer(
     contextFactory: SrtpContextFactory,
     parentLogger: Logger
 ) : SrtpTransformer(contextFactory, parentLogger) {
+    var earlyDiscardedPacketsSinceLastSuccess = 0
+    private val alwaysProcess = maxConsecutivePacketsDiscardedEarly <= 0
 
     override fun transform(packetInfo: PacketInfo, context: SrtpCryptoContext): SrtpErrorStatus {
-        // For silence packets we update the ROC (if authentication passes), but don't decrypt
-        return context.reverseTransformPacket(packetInfo.packetAs(), packetInfo.shouldDiscard).apply {
-            packetInfo.resetPayloadVerification()
+        // We want to avoid authenticating and decrypting packets that we are going to discarded (e.g. silence). We
+        // can not just discard them without passing them to the SRTP stack, because this will eventually break the ROC.
+        // Here we bypass the SRTP stack for packets marked to be discarded, but make sure that we haven't dropped too
+        // many consecutive packets.
+        if (packetInfo.shouldDiscard) {
+            return if (alwaysProcess ||
+                earlyDiscardedPacketsSinceLastSuccess++ > maxConsecutivePacketsDiscardedEarly) {
+                doTransform(packetInfo, context)
+            } else {
+                // Bypass the SRTP stack. The packet is already marked to be discarded, so there's no error condition.
+                SrtpErrorStatus.OK
+            }
         }
+
+        return doTransform(packetInfo, context)
     }
+
+    private fun doTransform(packetInfo: PacketInfo, context: SrtpCryptoContext): SrtpErrorStatus =
+        context.reverseTransformPacket(packetInfo.packetAs(), packetInfo.shouldDiscard).also {
+            packetInfo.resetPayloadVerification()
+            if (it == SrtpErrorStatus.OK) {
+                earlyDiscardedPacketsSinceLastSuccess = 0
+            }
+        }
 }
 
 /**
diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index e06f24655..2902c2b99 100644
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -17,6 +17,12 @@ jmt {
       }
     }
   }
+  srtp {
+    // The maximum number of packets that can be discarded early (without going through the SRTP stack for
+    // authentication), or -1 to authenticate all packets.
+    // This is useful to avoid unnecessary computation for audio silence.
+    max-consecutive-packets-discarded-early=-1
+  }
 
   transceiver {
       recv {