diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index a71925c5c6a852..30744553d16da8 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -735,7 +735,8 @@ void elf::riscvFinalizeRelax(int passes) {
       uint8_t *p = context().bAlloc.Allocate<uint8_t>(newSize);
       uint64_t offset = 0;
       int64_t delta = 0;
-      sec->rawData = makeArrayRef(p, newSize);
+      sec->content_ = p;
+      sec->size = newSize;
       sec->bytesDropped = 0;
 
       // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 40f60f9577a218..f08db553625407 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -55,7 +55,7 @@ InputSectionBase::InputSectionBase(InputFile *file, uint64_t flags,
                                    StringRef name, Kind sectionKind)
     : SectionBase(sectionKind, name, flags, entsize, alignment, type, info,
                   link),
-      file(file), rawData(data) {
+      file(file), content_(data.data()), size(data.size()) {
   // In order to reduce memory allocation, we assume that mergeable
   // sections are smaller than 4 GiB, which is not an unreasonable
   // assumption as of 2017.
@@ -111,9 +111,9 @@ size_t InputSectionBase::getSize() const {
 template <class ELFT>
 static void decompressAux(const InputSectionBase &sec, uint8_t *out,
                           size_t size) {
-  auto *hdr =
-      reinterpret_cast<const typename ELFT::Chdr *>(sec.content().data());
-  auto compressed = sec.content().slice(sizeof(typename ELFT::Chdr));
+  auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(sec.content_);
+  auto compressed = ArrayRef<uint8_t>(sec.content_, sec.compressedSize)
+                        .slice(sizeof(typename ELFT::Chdr));
   if (Error e = hdr->ch_type == ELFCOMPRESS_ZLIB
                     ? compression::zlib::decompress(compressed, out, size)
                     : compression::zstd::decompress(compressed, out, size))
@@ -130,7 +130,7 @@ void InputSectionBase::decompress() const {
   }
 
   invokeELFT(decompressAux, *this, uncompressedBuf, size);
-  rawData = makeArrayRef(uncompressedBuf, size);
+  content_ = uncompressedBuf;
   compressed = false;
 }
 
@@ -231,6 +231,7 @@ template <typename ELFT> void InputSectionBase::parseCompressedHeader() {
   }
 
   compressed = true;
+  compressedSize = size;
   size = hdr->ch_size;
   alignment = std::max<uint32_t>(hdr->ch_addralign, 1);
 }
@@ -1104,8 +1105,9 @@ template <class ELFT> void InputSection::writeTo(uint8_t *buf) {
   // If this is a compressed section, uncompress section contents directly
   // to the buffer.
   if (compressed) {
-    auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(content().data());
-    auto compressed = content().slice(sizeof(typename ELFT::Chdr));
+    auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(content_);
+    auto compressed = ArrayRef<uint8_t>(content_, compressedSize)
+                          .slice(sizeof(typename ELFT::Chdr));
     size_t size = this->size;
     if (Error e = hdr->ch_type == ELFCOMPRESS_ZLIB
                       ? compression::zlib::decompress(compressed, buf, size)
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index d433dfc70e8000..ed3f98a42f906e 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -155,20 +155,23 @@ class InputSectionBase : public SectionBase {
     bytesDropped -= num;
   }
 
-  mutable ArrayRef<uint8_t> rawData;
+  mutable const uint8_t *content_;
+  uint64_t size;
 
   void trim() {
     if (bytesDropped) {
-      rawData = rawData.drop_back(bytesDropped);
+      size -= bytesDropped;
       bytesDropped = 0;
     }
   }
 
-  ArrayRef<uint8_t> content() const { return rawData; }
+  ArrayRef<uint8_t> content() const {
+    return ArrayRef<uint8_t>(content_, size);
+  }
   ArrayRef<uint8_t> contentMaybeDecompress() const {
     if (compressed)
       decompress();
-    return rawData;
+    return content();
   }
 
   // The next member in the section group if this section is in a group. This is
@@ -217,6 +220,9 @@ class InputSectionBase : public SectionBase {
     // Auxiliary information for RISC-V linker relaxation. RISC-V does not use
     // jumpInstrMod.
     RISCVRelaxAux *relaxAux;
+
+    // The compressed content size when `compressed` is true.
+    size_t compressedSize;
   };
 
   // A function compiled with -fsplit-stack calling a function
@@ -238,12 +244,6 @@ class InputSectionBase : public SectionBase {
   template <typename ELFT>
   void parseCompressedHeader();
   void decompress() const;
-
-  // This field stores the uncompressed size of the compressed data in rawData,
-  // or -1 if rawData is not compressed (either because the section wasn't
-  // compressed in the first place, or because we ended up uncompressing it).
-  // Since the feature is not used often, this is usually -1.
-  mutable int64_t size = -1;
 };
 
 // SectionPiece represents a piece of splittable section contents.
@@ -397,7 +397,7 @@ class InputSection : public InputSectionBase {
   template <class ELFT> void copyShtGroup(uint8_t *buf);
 };
 
-static_assert(sizeof(InputSection) <= 160, "InputSection is too big");
+static_assert(sizeof(InputSection) <= 152, "InputSection is too big");
 
 class SyntheticSection : public InputSection {
 public: