diff --git a/index.js b/index.js
index e62a446..6542849 100644
--- a/index.js
+++ b/index.js
@@ -1019,36 +1019,59 @@ function asciiWrite (buf, string, offset, length) {
 }
 
 function base64Write (buf, string, offset, length) {
-  const src = string.replace(/[^+/0-9A-Za-z-_=]/g, '')
-  const eq = src.indexOf('=')
+  const src = string
   const dst = buf
 
-  let srcLen = eq >= 0 ? eq : src.length
+  let srcLen = src.length
   let srcPos = 0
   let dstLen = length
   let dstPos = offset
+  let invalid = false
+
+  if (srcLen > 0 && src[srcLen - 1] === '=') srcLen -= 1
+  if (srcLen > 0 && src[srcLen - 1] === '=') srcLen -= 1
+
+  retry: for (;;) { // eslint-disable-line
+    while (srcLen >= 4 && dstLen >= 3) {
+      const c1 = src.charCodeAt(srcPos++)
+      const c2 = src.charCodeAt(srcPos++)
+      const c3 = src.charCodeAt(srcPos++)
+      const c4 = src.charCodeAt(srcPos++)
+      const t1 = base64Table[c1 & 0x7f]
+      const t2 = base64Table[c2 & 0x7f]
+      const t3 = base64Table[c3 & 0x7f]
+      const t4 = base64Table[c4 & 0x7f]
+
+      if ((c1 | c2 | c3 | c4 | t1 | t2 | t3 | t4) & ~0x7f) {
+        srcPos -= 4
+        invalid = true // Slow parsing until we recover.
+        break
+      }
 
-  while (srcLen >= 4 && dstLen >= 3) {
-    const t1 = base64Table[src.charCodeAt(srcPos++)]
-    const t2 = base64Table[src.charCodeAt(srcPos++)]
-    const t3 = base64Table[src.charCodeAt(srcPos++)]
-    const t4 = base64Table[src.charCodeAt(srcPos++)]
-
-    dst[dstPos++] = (t1 << 2) | (t2 >> 4)
-    dst[dstPos++] = (t2 << 4) | (t3 >> 2)
-    dst[dstPos++] = (t3 << 6) | (t4 >> 0)
+      dst[dstPos++] = (t1 << 2) | (t2 >> 4)
+      dst[dstPos++] = (t2 << 4) | (t3 >> 2)
+      dst[dstPos++] = (t3 << 6) | (t4 >> 0)
 
-    srcLen -= 4
-    dstLen -= 3
-  }
+      srcLen -= 4
+      dstLen -= 3
+    }
 
-  {
     let w1, w2, w3, w4
+    let state = 0
 
     while (srcLen && dstLen) {
-      const w = base64Table[src.charCodeAt(srcPos)]
+      const c = src.charCodeAt(srcPos)
+      const w = base64Table[c & 0x7f]
+
+      srcPos++
+      srcLen--
 
-      switch (srcPos & 3) {
+      if ((c | w) & ~0x7f) {
+        if (c === 0x3d) break // Stop on `=`
+        continue // Skip invalid
+      }
+
+      switch (state & 3) {
         case 0:
           w1 = w
           break
@@ -1066,12 +1089,18 @@ function base64Write (buf, string, offset, length) {
           w4 = w
           dst[dstPos++] = (w3 << 6) | (w4 >> 0)
           dstLen--
+          if (invalid) {
+            // Retry fast parsing.
+            invalid = false
+            continue retry // eslint-disable-line
+          }
           break
       }
 
-      srcPos++
-      srcLen--
+      state++
     }
+
+    break
   }
 
   return dstPos - offset