diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
index 8a2c896140838..e774321c95e2d 100644
--- a/src/compiler/scanner.ts
+++ b/src/compiler/scanner.ts
@@ -1795,12 +1795,14 @@ export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean
 
             const ch = codePointAt(text, pos);
             if (pos === 0) {
-                // If a file wasn't valid text at all, it will usually be apparent at
-                // position 0 because UTF-8 decode will fail and produce U+FFFD.
+                // If a file isn't valid text at all, it will usually be apparent
+                // in the first few characters because UTF-8 decode will fail and produce U+FFFD.
                 // If that happens, just issue one error and refuse to try to scan further;
-                // this is likely a binary file that cannot be parsed
-                if (ch === CharacterCodes.replacementCharacter) {
-                    // Jump to the end of the file and fail.
+                // this is likely a binary file that cannot be parsed.
+                //
+                // It's safe to slice the text; U+FFFD can only be produced by an invalid decode,
+                // so even if we cut a surrogate pair in half, they wouldn't be U+FFFD.
+                if (text.slice(0, 256).includes("\uFFFD")) {
                     error(Diagnostics.File_appears_to_be_binary);
                     pos = end;
                     return token = SyntaxKind.NonTextFileMarkerTrivia;
diff --git a/src/compiler/types.ts b/src/compiler/types.ts
index 23873de4824a8..66c55690e6280 100644
--- a/src/compiler/types.ts
+++ b/src/compiler/types.ts
@@ -7531,9 +7531,6 @@ export const enum CharacterCodes {
     mathematicalSpace = 0x205F,
     ogham = 0x1680,
 
-    // Unicode replacement character produced when a byte sequence is invalid
-    replacementCharacter = 0xFFFD,
-
     _ = 0x5F,
     $ = 0x24,
 
diff --git a/tests/baselines/reference/TransportStream.errors.txt b/tests/baselines/reference/TransportStream.errors.txt
new file mode 100644
index 0000000000000..43b9b2755aeb9
--- /dev/null
+++ b/tests/baselines/reference/TransportStream.errors.txt
@@ -0,0 +1,7 @@
+TransportStream.ts(1,1): error TS1490: File appears to be binary.
+
+
+==== TransportStream.ts (1 errors) ====
+    G@�G@�G@�
+    
+!!! error TS1490: File appears to be binary.
\ No newline at end of file
diff --git a/tests/baselines/reference/TransportStream.js b/tests/baselines/reference/TransportStream.js
new file mode 100644
index 0000000000000..f73d86a37caab
--- /dev/null
+++ b/tests/baselines/reference/TransportStream.js
@@ -0,0 +1,6 @@
+//// [tests/cases/compiler/TransportStream.ts] ////
+
+//// [TransportStream.ts]
+G@�G@�G@�
+
+//// [TransportStream.js]
diff --git a/tests/baselines/reference/TransportStream.symbols b/tests/baselines/reference/TransportStream.symbols
new file mode 100644
index 0000000000000..e08ae8ea2ccd7
--- /dev/null
+++ b/tests/baselines/reference/TransportStream.symbols
@@ -0,0 +1,5 @@
+//// [tests/cases/compiler/TransportStream.ts] ////
+
+=== TransportStream.ts ===
+
+G@�G@�G@�
diff --git a/tests/baselines/reference/TransportStream.types b/tests/baselines/reference/TransportStream.types
new file mode 100644
index 0000000000000..e08ae8ea2ccd7
--- /dev/null
+++ b/tests/baselines/reference/TransportStream.types
@@ -0,0 +1,5 @@
+//// [tests/cases/compiler/TransportStream.ts] ////
+
+=== TransportStream.ts ===
+
+G@�G@�G@�
diff --git a/tests/cases/compiler/TransportStream.ts b/tests/cases/compiler/TransportStream.ts
new file mode 100644
index 0000000000000..730d320bfeed1
--- /dev/null
+++ b/tests/cases/compiler/TransportStream.ts
@@ -0,0 +1 @@
+G@’G@’G@’
\ No newline at end of file