diff --git a/Cargo.lock b/Cargo.lock
index b1c3ea377efb..8ec4d558e1df 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2902,6 +2902,17 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
 
+[[package]]
+name = "smartstring"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29"
+dependencies = [
+ "autocfg",
+ "static_assertions",
+ "version_check",
+]
+
 [[package]]
 name = "smawk"
 version = "0.3.1"
@@ -3723,6 +3734,7 @@ dependencies = [
  "serde",
  "serde_json",
  "smallvec",
+ "smartstring",
  "stacker",
  "swc_atoms",
  "swc_common",
diff --git a/crates/swc_ecma_parser/Cargo.toml b/crates/swc_ecma_parser/Cargo.toml
index 6c00e088b2be..085e8508dbd5 100644
--- a/crates/swc_ecma_parser/Cargo.toml
+++ b/crates/swc_ecma_parser/Cargo.toml
@@ -30,6 +30,7 @@ lexical        = { version = "6.1.0", features = ["power-of-two"] }
 num-bigint     = "0.4"
 serde          = { version = "1", features = ["derive"] }
 smallvec       = "1.8.0"
+smartstring    = "1"
 swc_atoms      = { version = "0.4.39", path = "../swc_atoms" }
 swc_common     = { version = "0.29.35", path = "../swc_common" }
 swc_ecma_ast   = { version = "0.98.2", path = "../swc_ecma_ast" }
diff --git a/crates/swc_ecma_parser/src/lexer/number.rs b/crates/swc_ecma_parser/src/lexer/number.rs
index bba333b03cfd..7cd4050c6ab8 100644
--- a/crates/swc_ecma_parser/src/lexer/number.rs
+++ b/crates/swc_ecma_parser/src/lexer/number.rs
@@ -2,7 +2,7 @@
 //!
 //!
 //! See https://tc39.github.io/ecma262/#sec-literals-numeric-literals
-use std::fmt::Write;
+use std::{borrow::Cow, fmt::Write};
 
 use either::Either;
 use num_bigint::BigInt as BigIntValue;
@@ -153,11 +153,14 @@ impl<'a> Lexer<'a> {
                     raw_val.push_str(raw.0.as_ref().unwrap());
                 }
 
-                raw_val
-                    // Remove number separator from number
-                    .replace('_', "")
-                    .parse()
-                    .expect("failed to parse float using rust's impl")
+                // Remove number separator from number
+                if raw_val.contains('_') {
+                    Cow::Owned(raw_val.replace('_', ""))
+                } else {
+                    Cow::Borrowed(&raw_val)
+                }
+                .parse()
+                .expect("failed to parse float using rust's impl")
             };
         }
 
@@ -210,10 +213,13 @@ impl<'a> Lexer<'a> {
 
                     write!(raw_val, "{}", exp).unwrap();
 
-                    raw_val
-                        .replace('_', "")
-                        .parse()
-                        .expect("failed to parse float literal")
+                    if raw_val.contains('_') {
+                        Cow::Owned(raw_val.replace('_', ""))
+                    } else {
+                        Cow::Borrowed(&raw_val)
+                    }
+                    .parse()
+                    .expect("failed to parse float literal")
                 }
             }
             _ => {}