[AIX] support write operation of big archive.

SUMMARY

1. Enable supporting the write operation of big archive.
2. the first commit come from https://reviews.llvm.org/D104367
3. refactor the first commit and implement writing symbol table.
4. fixed the bugs and add more test cases in the second commit.

Reviewers: James Henderson
Differential Revision: https://reviews.llvm.org/D123949

Author: diggerlin

llvm/include/llvm/Object/Archive.h

@@ -391,6 +391,7 @@ class Archive : public Binary {
 };
 
 class BigArchive : public Archive {
+public:
   /// Fixed-Length Header.
   struct FixLenHdr {
     char Magic[sizeof(BigArchiveMagic) - 1]; ///< Big archive magic string.

llvm/lib/Object/ArchiveWriter.cpp

@@ -25,6 +25,7 @@
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
@@ -127,16 +128,20 @@ static bool isDarwin(object::Archive::Kind Kind) {
          Kind == object::Archive::K_DARWIN64;
 }
 
+static bool isAIXBigArchive(object::Archive::Kind Kind) {
+  return Kind == object::Archive::K_AIXBIG;
+}
+
 static bool isBSDLike(object::Archive::Kind Kind) {
   switch (Kind) {
   case object::Archive::K_GNU:
   case object::Archive::K_GNU64:
+  case object::Archive::K_AIXBIG:
     return false;
   case object::Archive::K_BSD:
   case object::Archive::K_DARWIN:
   case object::Archive::K_DARWIN64:
     return true;
-  case object::Archive::K_AIXBIG:
   case object::Archive::K_COFF:
     break;
   }
@@ -189,6 +194,31 @@ printBSDMemberHeader(raw_ostream &Out, uint64_t Pos, StringRef Name,
     Out.write(uint8_t(0));
 }
 
+static void
+printBigArchiveMemberHeader(raw_ostream &Out, StringRef Name,
+                            const sys::TimePoint<std::chrono::seconds> &ModTime,
+                            unsigned UID, unsigned GID, unsigned Perms,
+                            uint64_t Size, unsigned PrevOffset,
+                            unsigned NextOffset) {
+  unsigned NameLen = Name.size();
+
+  printWithSpacePadding(Out, Size, 20);           // File member size
+  printWithSpacePadding(Out, NextOffset, 20);     // Next member header offset
+  printWithSpacePadding(Out, PrevOffset, 20); // Previous member header offset
+  printWithSpacePadding(Out, sys::toTimeT(ModTime), 12); // File member date
+  // The big archive format has 12 chars for uid and gid.
+  printWithSpacePadding(Out, UID % 1000000000000, 12);   // UID
+  printWithSpacePadding(Out, GID % 1000000000000, 12);   // GID
+  printWithSpacePadding(Out, format("%o", Perms), 12);   // Permission
+  printWithSpacePadding(Out, NameLen, 4);                // Name length
+  if (NameLen) {
+    printWithSpacePadding(Out, Name, NameLen); // Name
+    if (NameLen % 2)
+      Out.write(uint8_t(0)); // Null byte padding
+  }
+  Out << "`\n"; // Terminator
+}
+
 static bool useStringTable(bool Thin, StringRef Name) {
   return Thin || Name.size() >= 16 || Name.contains('/');
 }
@@ -199,8 +229,8 @@ static bool is64BitKind(object::Archive::Kind Kind) {
   case object::Archive::K_BSD:
   case object::Archive::K_DARWIN:
   case object::Archive::K_COFF:
-  case object::Archive::K_AIXBIG:
     return false;
+  case object::Archive::K_AIXBIG:
   case object::Archive::K_DARWIN64:
   case object::Archive::K_GNU64:
     return true;
@@ -304,19 +334,27 @@ static uint64_t computeSymbolTableSize(object::Archive::Kind Kind,
   // least 4-byte aligned for 32-bit content.  Opt for the larger encoding
   // uniformly.
   // We do this for all bsd formats because it simplifies aligning members.
-  uint32_t Pad = offsetToAlignment(Size, Align(isBSDLike(Kind) ? 8 : 2));
+  // For the big archive format, the symbol table is the last member, so there
+  // is no need to align.
+  uint32_t Pad = isAIXBigArchive(Kind)
+                     ? 0
+                     : offsetToAlignment(Size, Align(isBSDLike(Kind) ? 8 : 2));
   Size += Pad;
   if (Padding)
     *Padding = Pad;
   return Size;
 }
 
 static void writeSymbolTableHeader(raw_ostream &Out, object::Archive::Kind Kind,
-                                   bool Deterministic, uint64_t Size) {
+                                   bool Deterministic, uint64_t Size,
+                                   uint64_t PrevMemberOffset = 0) {
   if (isBSDLike(Kind)) {
     const char *Name = is64BitKind(Kind) ? "__.SYMDEF_64" : "__.SYMDEF";
     printBSDMemberHeader(Out, Out.tell(), Name, now(Deterministic), 0, 0, 0,
                          Size);
+  } else if (isAIXBigArchive(Kind)) {
+    printBigArchiveMemberHeader(Out, "", now(Deterministic), 0, 0,
+                                0, Size, PrevMemberOffset, 0);
   } else {
     const char *Name = is64BitKind(Kind) ? "/SYM64" : "";
     printGNUSmallMemberHeader(Out, Name, now(Deterministic), 0, 0, 0, Size);
@@ -325,7 +363,8 @@ static void writeSymbolTableHeader(raw_ostream &Out, object::Archive::Kind Kind,
 
 static void writeSymbolTable(raw_ostream &Out, object::Archive::Kind Kind,
                              bool Deterministic, ArrayRef<MemberData> Members,
-                             StringRef StringTable) {
+                             StringRef StringTable,
+                             uint64_t PrevMemberOffset = 0) {
   // We don't write a symbol table on an archive with no members -- except on
   // Darwin, where the linker will abort unless the archive has a symbol table.
   if (StringTable.empty() && !isDarwin(Kind))
@@ -338,9 +377,10 @@ static void writeSymbolTable(raw_ostream &Out, object::Archive::Kind Kind,
   uint64_t OffsetSize = is64BitKind(Kind) ? 8 : 4;
   uint32_t Pad;
   uint64_t Size = computeSymbolTableSize(Kind, NumSyms, OffsetSize, StringTable, &Pad);
-  writeSymbolTableHeader(Out, Kind, Deterministic, Size);
+  writeSymbolTableHeader(Out, Kind, Deterministic, Size, PrevMemberOffset);
 
-  uint64_t Pos = Out.tell() + Size;
+  uint64_t Pos = isAIXBigArchive(Kind) ? sizeof(object::BigArchive::FixLenHdr)
+                                       : Out.tell() + Size;
 
   if (isBSDLike(Kind))
     printNBits(Out, Kind, NumSyms * 2 * OffsetSize);
@@ -409,9 +449,8 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
                   bool NeedSymbols, ArrayRef<NewArchiveMember> NewMembers) {
   static char PaddingData[8] = {'\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'};
 
-  // This ignores the symbol table, but we only need the value mod 8 and the
-  // symbol table is aligned to be a multiple of 8 bytes
-  uint64_t Pos = 0;
+  uint64_t Pos =
+      isAIXBigArchive(Kind) ? sizeof(object::BigArchive::FixLenHdr) : 0;
 
   std::vector<MemberData> Ret;
   bool HasObject = false;
@@ -471,6 +510,9 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
       Entry.second = Entry.second > 1 ? 1 : 0;
   }
 
+  // The big archive format needs to know the offset of the previous member
+  // header.
+  unsigned PrevOffset = 0;
   for (const NewArchiveMember &M : NewMembers) {
     std::string Header;
     raw_string_ostream Out(Header);
@@ -503,8 +545,16 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
           std::move(StringMsg), object::object_error::parse_failed);
     }
 
-    printMemberHeader(Out, Pos, StringTable, MemberNames, Kind, Thin, M,
-                      ModTime, Size);
+    if (isAIXBigArchive(Kind)) {
+      unsigned NextOffset = Pos + sizeof(object::BigArMemHdrType) +
+                            alignTo(M.MemberName.size(), 2) + alignTo(Size, 2);
+      printBigArchiveMemberHeader(Out, M.MemberName, ModTime, M.UID, M.GID,
+                                  M.Perms, Size, PrevOffset, NextOffset);
+      PrevOffset = Pos;
+    } else {
+      printMemberHeader(Out, Pos, StringTable, MemberNames, Kind, Thin, M,
+                        ModTime, Size);
+    }
     Out.flush();
 
     std::vector<unsigned> Symbols;
@@ -588,22 +638,25 @@ static Error writeArchiveToStream(raw_ostream &Out,
     return E;
   std::vector<MemberData> &Data = *DataOrErr;
 
-  if (!StringTableBuf.empty())
+  if (!StringTableBuf.empty() && !isAIXBigArchive(Kind))
     Data.insert(Data.begin(), computeStringTable(StringTableBuf));
 
   // We would like to detect if we need to switch to a 64-bit symbol table.
-  if (WriteSymtab) {
-    uint64_t MaxOffset = 8; // For the file signature.
-    uint64_t LastOffset = MaxOffset;
-    uint64_t NumSyms = 0;
-    for (const auto &M : Data) {
-      // Record the start of the member's offset
-      LastOffset = MaxOffset;
-      // Account for the size of each part associated with the member.
-      MaxOffset += M.Header.size() + M.Data.size() + M.Padding.size();
-      NumSyms += M.Symbols.size();
-    }
+  uint64_t LastMemberEndOffset =
+      isAIXBigArchive(Kind) ? sizeof(object::BigArchive::FixLenHdr) : 8;
+  uint64_t LastMemberHeaderOffset = LastMemberEndOffset;
+  uint64_t NumSyms = 0;
+  for (const auto &M : Data) {
+    // Record the start of the member's offset
+    LastMemberHeaderOffset = LastMemberEndOffset;
+    // Account for the size of each part associated with the member.
+    LastMemberEndOffset += M.Header.size() + M.Data.size() + M.Padding.size();
+    NumSyms += M.Symbols.size();
+  }
 
+  // The symbol table is put at the end of the big archive file. The symbol
+  // table is at the start of the archive file for other archive formats.
+  if (WriteSymtab && !isAIXBigArchive(Kind)) {
     // We assume 32-bit offsets to see if 32-bit symbols are possible or not.
     uint64_t SymtabSize = computeSymbolTableSize(Kind, NumSyms, 4, SymNamesBuf);
     auto computeSymbolTableHeaderSize =
@@ -613,7 +666,7 @@ static Error writeArchiveToStream(raw_ostream &Out,
           writeSymbolTableHeader(Tmp, Kind, Deterministic, SymtabSize);
           return TmpBuf.size();
         };
-    LastOffset += computeSymbolTableHeaderSize() + SymtabSize;
+    LastMemberHeaderOffset += computeSymbolTableHeaderSize() + SymtabSize;
 
     // The SYM64 format is used when an archive's member offsets are larger than
     // 32-bits can hold. The need for this shift in format is detected by
@@ -627,10 +680,10 @@ static Error writeArchiveToStream(raw_ostream &Out,
     if (Sym64Env)
       StringRef(Sym64Env).getAsInteger(10, Sym64Threshold);
 
-    // If LastOffset isn't going to fit in a 32-bit varible we need to switch
-    // to 64-bit. Note that the file can be larger than 4GB as long as the last
-    // member starts before the 4GB offset.
-    if (LastOffset >= Sym64Threshold) {
+    // If LastMemberHeaderOffset isn't going to fit in a 32-bit varible we need
+    // to switch to 64-bit. Note that the file can be larger than 4GB as long as
+    // the last member starts before the 4GB offset.
+    if (LastMemberHeaderOffset >= Sym64Threshold) {
       if (Kind == object::Archive::K_DARWIN)
         Kind = object::Archive::K_DARWIN64;
       else
@@ -640,15 +693,92 @@ static Error writeArchiveToStream(raw_ostream &Out,
 
   if (Thin)
     Out << "!<thin>\n";
+  else if (isAIXBigArchive(Kind))
+    Out << "<bigaf>\n";
   else
     Out << "!<arch>\n";
 
-  if (WriteSymtab)
-    writeSymbolTable(Out, Kind, Deterministic, Data, SymNamesBuf);
+  if (!isAIXBigArchive(Kind)) {
+    if (WriteSymtab)
+      writeSymbolTable(Out, Kind, Deterministic, Data, SymNamesBuf);
+    for (const MemberData &M : Data)
+      Out << M.Header << M.Data << M.Padding;
+  } else {
+    // For the big archive (AIX) format, compute a table of member names and
+    // offsets, used in the member table.
+    uint64_t MemberTableNameStrTblSize = 0;
+    std::vector<size_t> MemberOffsets;
+    std::vector<StringRef> MemberNames;
+    // Loop across object to find offset and names.
+    uint64_t MemberEndOffset = sizeof(object::BigArchive::FixLenHdr);
+    for (size_t I = 0, Size = NewMembers.size(); I != Size; ++I) {
+      const NewArchiveMember &Member = NewMembers[I];
+      MemberTableNameStrTblSize += Member.MemberName.size() + 1;
+      MemberOffsets.push_back(MemberEndOffset);
+      MemberNames.push_back(Member.MemberName);
+      // File member name ended with "`\n". The length is included in
+      // BigArMemHdrType.
+      MemberEndOffset += sizeof(object::BigArMemHdrType) +
+                             alignTo(Data[I].Data.size(), 2) +
+                             alignTo(Member.MemberName.size(), 2);
+    }
 
-  for (const MemberData &M : Data)
-    Out << M.Header << M.Data << M.Padding;
+    // AIX member table size.
+    unsigned MemberTableSize = 20 + // Number of members field
+                               20 * MemberOffsets.size() +
+                               MemberTableNameStrTblSize;
+
+    unsigned GlobalSymbolOffset =
+        (WriteSymtab && NumSyms > 0)
+            ? LastMemberEndOffset +
+                  alignTo(sizeof(object::BigArMemHdrType) + MemberTableSize, 2)
+            : 0;
+
+    // Fixed Sized Header.
+    printWithSpacePadding(Out, NewMembers.size() ? LastMemberEndOffset : 0,
+                          20); // Offset to member table
+    // If there are no file members in the archive, there will be no global
+    // symbol table.
+    printWithSpacePadding(Out, NewMembers.size() ? GlobalSymbolOffset : 0, 20);
+    printWithSpacePadding(
+        Out, 0,
+        20); // Offset to 64 bits global symbol table - Not supported yet
+    printWithSpacePadding(
+        Out, NewMembers.size() ? sizeof(object::BigArchive::FixLenHdr) : 0,
+        20); // Offset to first archive member
+    printWithSpacePadding(Out, NewMembers.size() ? LastMemberHeaderOffset : 0,
+                          20); // Offset to last archive member
+    printWithSpacePadding(
+        Out, 0,
+        20); // Offset to first member of free list - Not supported yet
+
+    for (const MemberData &M : Data) {
+      Out << M.Header << M.Data;
+      if (M.Data.size() % 2)
+        Out << '\0';
+    }
 
+    if (NewMembers.size()) {
+      // Member table.
+      printBigArchiveMemberHeader(Out, "", sys::toTimePoint(0), 0, 0, 0,
+                                  MemberTableSize, LastMemberHeaderOffset,
+                                  GlobalSymbolOffset);
+      printWithSpacePadding(Out, MemberOffsets.size(), 20); // Number of members
+      for (uint64_t MemberOffset : MemberOffsets)
+        printWithSpacePadding(Out, MemberOffset,
+                              20); // Offset to member file header.
+      for (StringRef MemberName : MemberNames)
+        Out << MemberName << '\0'; // Member file name, null byte padding.
+
+      if (MemberTableNameStrTblSize % 2)
+        Out << '\0'; // Name table must be tail padded to an even number of
+                     // bytes.
+
+      if (WriteSymtab && NumSyms > 0)
+        writeSymbolTable(Out, Kind, Deterministic, Data, SymNamesBuf,
+                         LastMemberEndOffset);
+    }
+  }
   Out.flush();
   return Error::success();
 }

llvm/test/Object/ar-create.test

@@ -1,4 +1,3 @@
-XFAIL: system-aix
 Test which operations create an archive and which don't.
 
 RUN: touch %t

llvm/test/Object/archive-extract-dir.test

@@ -1,5 +1,3 @@
-XFAIL: system-aix
-
 RUN: mkdir -p %t
 RUN: cd %t
 RUN: rm -rf foo

llvm/test/Object/archive-malformed-object.test

@@ -1,4 +1,3 @@
-# XFAIL: system-aix
 ## Show that the archive library emits error messages when adding malformed
 ## objects.
 

llvm/test/Object/archive-replace-pos.test

@@ -1,5 +1,3 @@
-XFAIL: system-aix
-
 Test adding a member to a particular position
 
 RUN: touch %t.foo

llvm/test/Object/archive-unknown-filetype.test

@@ -1,4 +1,3 @@
-# XFAIL: system-aix
 ## Show that the archive library does not emit an error or add any symbols to
 ## the archive symbol table, when it encounters an unknown file type, but still
 ## adds the file to the archive.

llvm/test/Object/archive-update.test

@@ -1,5 +1,3 @@
-XFAIL: system-aix
-
 Test the 'u' option of llvm-ar
 
 RUN: rm -rf %t && mkdir -p %t && cd %t

llvm/test/Object/directory.ll

@@ -1,5 +1,3 @@
-; XFAIL: system-aix
-
 ;RUN: rm -rf %t && mkdir -p %t
 ;RUN: not llvm-ar r %t/test.a . 2>&1 | FileCheck -DMSG=%errc_EISDIR %s
 ;CHECK: .: [[MSG]]

llvm/test/tools/llvm-ar/count.test

@@ -1,4 +1,3 @@
-# XFAIL: system-aix
 # Test the 'N' count parameter.
 
 # Get a temp clean cwd to extract into.

llvm/test/tools/llvm-ar/create.test

@@ -1,4 +1,3 @@
-XFAIL: system-aix
 ## Test the creation warning and supression of that warning.
 
 RUN: touch %t1.txt

llvm/test/tools/llvm-ar/dash-before-letter.test

@@ -1,4 +1,3 @@
-XFAIL: system-aix
 # Test the use of dash before key letters.
 
 RUN: touch %t1.txt