Skip to content

Commit

Permalink
Clang Driver: refactor support for writing response files to be
Browse files Browse the repository at this point in the history 
specified at Command creation, rather than as part of the Tool.

This resolves the hack I just added to allow Darwin toolchain to vary
its level of support based on `-mlinker-version=`.

The change preserves the _current_ settings for response-file support.
Some tools look likely to be declaring that they don't support
response files in error, however I kept them as-is in order for this
change to be a simple refactoring.

Differential Revision: https://reviews.llvm.org/D82782
  • Loading branch information
@jyknight
jyknight committed Jun 29, 2020
1 parent 381df16 commit 4772b99
Show file tree
Hide file tree
Showing 61 changed files with 349 additions and 299 deletions.
85 changes: 80 additions & 5 deletions clang/include/clang/Driver/Job.h
View file
Expand Up @@ -16,6 +16,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator.h"
#include "llvm/Option/Option.h"
#include "llvm/Support/Program.h"
#include <memory>
#include <string>
#include <utility>
Expand All @@ -36,6 +37,69 @@ struct CrashReportInfo {
: Filename(Filename), VFSPath(VFSPath) {}
};

// Encodes the kind of response file supported for a command invocation.
// Response files are necessary if the command line gets too large, requiring
// the arguments to be transferred to a file.
struct ResponseFileSupport {
enum ResponseFileKind {
// Provides full support for response files, which means we can transfer
// all tool input arguments to a file.
RF_Full,
// Input file names can live in a file, but flags can't. This is a special
// case for old versions of Apple's ld64.
RF_FileList,
// Does not support response files: all arguments must be passed via
// command line.
RF_None
};
/// The level of support for response files.
ResponseFileKind ResponseKind;

/// The encoding to use when writing response files on Windows. Ignored on
/// other host OSes.
///
/// Windows use cases: - GCC and Binutils on mingw only accept ANSI response
/// files encoded with the system current code page.
/// - MSVC's CL.exe and LINK.exe accept UTF16 on Windows.
/// - Clang accepts both UTF8 and UTF16.
///
/// FIXME: When GNU tools learn how to parse UTF16 on Windows, we should
/// always use UTF16 for Windows, which is the Windows official encoding for
/// international characters.
llvm::sys::WindowsEncodingMethod ResponseEncoding;

/// What prefix to use for the command-line argument when passing a response
/// file.
const char *ResponseFlag;

/// Returns a ResponseFileSupport indicating that response files are not
/// supported.
static constexpr ResponseFileSupport None() {
return {RF_None, llvm::sys::WEM_UTF8, nullptr};
}

/// Returns a ResponseFileSupport indicating that response files are
/// supported, using the @file syntax. On windows, the file is written in the
/// UTF8 encoding. On other OSes, no re-encoding occurs.
static constexpr ResponseFileSupport AtFileUTF8() {
return {RF_Full, llvm::sys::WEM_UTF8, "@"};
}

/// Returns a ResponseFileSupport indicating that response files are
/// supported, using the @file syntax. On windows, the file is written in the
/// current ANSI code-page encoding. On other OSes, no re-encoding occurs.
static constexpr ResponseFileSupport AtFileCurCP() {
return {RF_Full, llvm::sys::WEM_CurrentCodePage, "@"};
}

/// Returns a ResponseFileSupport indicating that response files are
/// supported, using the @file syntax. On windows, the file is written in the
/// UTF-16 encoding. On other OSes, no re-encoding occurs.
static constexpr ResponseFileSupport AtFileUTF16() {
return {RF_Full, llvm::sys::WEM_UTF16, "@"};
}
};

/// Command - An executable path/name and argument vector to
/// execute.
class Command {
Expand All @@ -45,6 +109,9 @@ class Command {
/// Tool - The tool which caused the creation of this job.
const Tool &Creator;

/// Whether and how to generate response files if the arguments are too long.
ResponseFileSupport ResponseSupport;

/// The executable to run.
const char *Executable;

Expand Down Expand Up @@ -89,7 +156,8 @@ class Command {
/// Whether the command will be executed in this process or not.
bool InProcess = false;

Command(const Action &Source, const Tool &Creator, const char *Executable,
Command(const Action &Source, const Tool &Creator,
ResponseFileSupport ResponseSupport, const char *Executable,
const llvm::opt::ArgStringList &Arguments,
ArrayRef<InputInfo> Inputs);
// FIXME: This really shouldn't be copyable, but is currently copied in some
Expand All @@ -109,11 +177,16 @@ class Command {
/// getCreator - Return the Tool which caused the creation of this job.
const Tool &getCreator() const { return Creator; }

/// Returns the kind of response file supported by the current invocation.
const ResponseFileSupport &getResponseFileSupport() {
return ResponseSupport;
}

/// Set to pass arguments via a response file when launching the command
void setResponseFile(const char *FileName);

/// Set an input file list, necessary if we need to use a response file but
/// the tool being called only supports input files lists.
/// Set an input file list, necessary if you specified an RF_FileList response
/// file support.
void setInputFileList(llvm::opt::ArgStringList List) {
InputFileList = std::move(List);
}
Expand All @@ -136,7 +209,8 @@ class Command {
/// Use the CC1 tool callback when available, to avoid creating a new process
class CC1Command : public Command {
public:
CC1Command(const Action &Source, const Tool &Creator, const char *Executable,
CC1Command(const Action &Source, const Tool &Creator,
ResponseFileSupport ResponseSupport, const char *Executable,
const llvm::opt::ArgStringList &Arguments,
ArrayRef<InputInfo> Inputs);

Expand All @@ -154,7 +228,7 @@ class CC1Command : public Command {
class FallbackCommand : public Command {
public:
FallbackCommand(const Action &Source_, const Tool &Creator_,
const char *Executable_,
ResponseFileSupport ResponseSupport, const char *Executable_,
const llvm::opt::ArgStringList &Arguments_,
ArrayRef<InputInfo> Inputs,
std::unique_ptr<Command> Fallback_);
Expand All @@ -173,6 +247,7 @@ class FallbackCommand : public Command {
class ForceSuccessCommand : public Command {
public:
ForceSuccessCommand(const Action &Source_, const Tool &Creator_,
ResponseFileSupport ResponseSupport,
const char *Executable_,
const llvm::opt::ArgStringList &Arguments_,
ArrayRef<InputInfo> Inputs);
Expand Down
56 changes: 1 addition & 55 deletions clang/include/clang/Driver/Tool.h
View file
Expand Up @@ -10,7 +10,6 @@
#define LLVM_CLANG_DRIVER_TOOL_H

#include "clang/Basic/LLVM.h"
#include "llvm/Support/Program.h"

namespace llvm {
namespace opt {
Expand All @@ -31,24 +30,6 @@ namespace driver {

/// Tool - Information on a specific compilation tool.
class Tool {
public:
// Documents the level of support for response files in this tool.
// Response files are necessary if the command line gets too large,
// requiring the arguments to be transferred to a file.
enum ResponseFileSupport {
// Provides full support for response files, which means we can transfer
// all tool input arguments to a file. E.g.: clang, gcc, binutils and MSVC
// tools.
RF_Full,
// Input file names can live in a file, but flags can't. E.g.: ld64 (Mac
// OS X linker).
RF_FileList,
// Does not support response files: all arguments must be passed via
// command line.
RF_None
};

private:
/// The tool name (for debugging).
const char *Name;

Expand All @@ -58,20 +39,8 @@ class Tool {
/// The tool chain this tool is a part of.
const ToolChain &TheToolChain;

/// The level of support for response files seen in this tool
const ResponseFileSupport ResponseSupport;

/// The encoding to use when writing response files for this tool on Windows
const llvm::sys::WindowsEncodingMethod ResponseEncoding;

/// The flag used to pass a response file via command line to this tool
const char *const ResponseFlag;

public:
Tool(const char *Name, const char *ShortName, const ToolChain &TC,
ResponseFileSupport ResponseSupport = RF_None,
llvm::sys::WindowsEncodingMethod ResponseEncoding = llvm::sys::WEM_UTF8,
const char *ResponseFlag = "@");
Tool(const char *Name, const char *ShortName, const ToolChain &TC);

public:
virtual ~Tool();
Expand All @@ -87,29 +56,6 @@ class Tool {
virtual bool hasIntegratedCPP() const = 0;
virtual bool isLinkJob() const { return false; }
virtual bool isDsymutilJob() const { return false; }
/// Returns the level of support for response files of this tool,
/// whether it accepts arguments to be passed via a file on disk.
ResponseFileSupport getResponseFilesSupport() const {
return ResponseSupport;
}
/// Returns which encoding the response file should use. This is only
/// relevant on Windows platforms where there are different encodings being
/// accepted for different tools. On UNIX, UTF8 is universal.
///
/// Windows use cases: - GCC and Binutils on mingw only accept ANSI response
/// files encoded with the system current code page.
/// - MSVC's CL.exe and LINK.exe accept UTF16 on Windows.
/// - Clang accepts both UTF8 and UTF16.
///
/// FIXME: When GNU tools learn how to parse UTF16 on Windows, we should
/// always use UTF16 for Windows, which is the Windows official encoding for
/// international characters.
llvm::sys::WindowsEncodingMethod getResponseFileEncoding() const {
return ResponseEncoding;
}
/// Returns which prefix to use when passing the name of a response
/// file as a parameter to this tool.
const char *getResponseFileFlag() const { return ResponseFlag; }

/// Does this tool have "good" standardized diagnostics, or should the
/// driver add an additional "command failed" diagnostic on failures.
Expand Down
3 changes: 0 additions & 3 deletions clang/include/clang/Driver/ToolChain.h
View file
Expand Up @@ -201,9 +201,6 @@ class ToolChain {

// Accessors

/// Temporary for Darwin::Linker
const llvm::opt::ArgList &getArgs_DO_NOT_USE() const { return Args; }

const Driver &getDriver() const { return D; }
llvm::vfs::FileSystem &getVFS() const;
const llvm::Triple &getTriple() const { return Triple; }
Expand Down
3 changes: 2 additions & 1 deletion clang/lib/Driver/Driver.cpp
View file
Expand Up @@ -1456,7 +1456,8 @@ void Driver::setUpResponseFiles(Compilation &C, Command &Cmd) {
// capacity if the tool does not support response files, there is a chance/
// that things will just work without a response file, so we silently just
// skip it.
if (Cmd.getCreator().getResponseFilesSupport() == Tool::RF_None ||
if (Cmd.getResponseFileSupport().ResponseKind ==
ResponseFileSupport::RF_None ||
llvm::sys::commandLineFitsWithinSystemLimits(Cmd.getExecutable(),
Cmd.getArguments()))
return;
Expand Down
31 changes: 18 additions & 13 deletions clang/lib/Driver/Job.cpp
View file
Expand Up @@ -36,11 +36,11 @@ using namespace clang;
using namespace driver;

Command::Command(const Action &Source, const Tool &Creator,
const char *Executable,
ResponseFileSupport ResponseSupport, const char *Executable,
const llvm::opt::ArgStringList &Arguments,
ArrayRef<InputInfo> Inputs)
: Source(Source), Creator(Creator), Executable(Executable),
Arguments(Arguments) {
: Source(Source), Creator(Creator), ResponseSupport(ResponseSupport),
Executable(Executable), Arguments(Arguments) {
for (const auto &II : Inputs)
if (II.isFilename())
InputFilenames.push_back(II.getFilename());
Expand Down Expand Up @@ -102,7 +102,7 @@ static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,

void Command::writeResponseFile(raw_ostream &OS) const {
// In a file list, we only write the set of inputs to the response file
if (Creator.getResponseFilesSupport() == Tool::RF_FileList) {
if (ResponseSupport.ResponseKind == ResponseFileSupport::RF_FileList) {
for (const auto *Arg : InputFileList) {
OS << Arg << '\n';
}
Expand Down Expand Up @@ -131,7 +131,7 @@ void Command::buildArgvForResponseFile(
// When not a file list, all arguments are sent to the response file.
// This leaves us to set the argv to a single parameter, requesting the tool
// to read the response file.
if (Creator.getResponseFilesSupport() != Tool::RF_FileList) {
if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList) {
Out.push_back(Executable);
Out.push_back(ResponseFileFlag.c_str());
return;
Expand All @@ -149,7 +149,7 @@ void Command::buildArgvForResponseFile(
Out.push_back(Arg);
} else if (FirstInput) {
FirstInput = false;
Out.push_back(Creator.getResponseFileFlag());
Out.push_back(ResponseSupport.ResponseFlag);
Out.push_back(ResponseFile);
}
}
Expand Down Expand Up @@ -277,7 +277,7 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,
writeResponseFile(OS);
// Avoiding duplicated newline terminator, since FileLists are
// newline-separated.
if (Creator.getResponseFilesSupport() != Tool::RF_FileList)
if (ResponseSupport.ResponseKind != ResponseFileSupport::RF_FileList)
OS << "\n";
OS << " (end of response file)";
}
Expand All @@ -287,7 +287,7 @@ void Command::Print(raw_ostream &OS, const char *Terminator, bool Quote,

void Command::setResponseFile(const char *FileName) {
ResponseFile = FileName;
ResponseFileFlag = Creator.getResponseFileFlag();
ResponseFileFlag = ResponseSupport.ResponseFlag;
ResponseFileFlag += FileName;
}

Expand Down Expand Up @@ -327,7 +327,7 @@ int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,

// Save the response file in the appropriate encoding
if (std::error_code EC = writeFileWithEncoding(
ResponseFile, RespContents, Creator.getResponseFileEncoding())) {
ResponseFile, RespContents, ResponseSupport.ResponseEncoding)) {
if (ErrMsg)
*ErrMsg = EC.message();
if (ExecutionFailed)
Expand All @@ -354,10 +354,11 @@ int Command::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
}

CC1Command::CC1Command(const Action &Source, const Tool &Creator,
ResponseFileSupport ResponseSupport,
const char *Executable,
const llvm::opt::ArgStringList &Arguments,
ArrayRef<InputInfo> Inputs)
: Command(Source, Creator, Executable, Arguments, Inputs) {
: Command(Source, Creator, ResponseSupport, Executable, Arguments, Inputs) {
InProcess = true;
}

Expand Down Expand Up @@ -410,11 +411,13 @@ void CC1Command::setEnvironment(llvm::ArrayRef<const char *> NewEnvironment) {
}

FallbackCommand::FallbackCommand(const Action &Source_, const Tool &Creator_,
ResponseFileSupport ResponseSupport,
const char *Executable_,
const llvm::opt::ArgStringList &Arguments_,
ArrayRef<InputInfo> Inputs,
std::unique_ptr<Command> Fallback_)
: Command(Source_, Creator_, Executable_, Arguments_, Inputs),
: Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_,
Inputs),
Fallback(std::move(Fallback_)) {}

void FallbackCommand::Print(raw_ostream &OS, const char *Terminator,
Expand Down Expand Up @@ -451,9 +454,11 @@ int FallbackCommand::Execute(ArrayRef<llvm::Optional<StringRef>> Redirects,
}

ForceSuccessCommand::ForceSuccessCommand(
const Action &Source_, const Tool &Creator_, const char *Executable_,
const Action &Source_, const Tool &Creator_,
ResponseFileSupport ResponseSupport, const char *Executable_,
const llvm::opt::ArgStringList &Arguments_, ArrayRef<InputInfo> Inputs)
: Command(Source_, Creator_, Executable_, Arguments_, Inputs) {}
: Command(Source_, Creator_, ResponseSupport, Executable_, Arguments_,
Inputs) {}

void ForceSuccessCommand::Print(raw_ostream &OS, const char *Terminator,
bool Quote, CrashReportInfo *CrashInfo) const {
Expand Down
9 changes: 2 additions & 7 deletions clang/lib/Driver/Tool.cpp
View file
Expand Up @@ -11,13 +11,8 @@

using namespace clang::driver;

Tool::Tool(const char *_Name, const char *_ShortName, const ToolChain &TC,
ResponseFileSupport _ResponseSupport,
llvm::sys::WindowsEncodingMethod _ResponseEncoding,
const char *_ResponseFlag)
: Name(_Name), ShortName(_ShortName), TheToolChain(TC),
ResponseSupport(_ResponseSupport), ResponseEncoding(_ResponseEncoding),
ResponseFlag(_ResponseFlag) {}
Tool::Tool(const char *_Name, const char *_ShortName, const ToolChain &TC)
: Name(_Name), ShortName(_ShortName), TheToolChain(TC) {}

Tool::~Tool() {
}
Expand Down
6 changes: 4 additions & 2 deletions clang/lib/Driver/ToolChains/AIX.cpp
View file
Expand Up @@ -73,7 +73,8 @@ void aix::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
CmdArgs.push_back(II.getFilename());

const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
Exec, CmdArgs, Inputs));
}

void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
Expand Down Expand Up @@ -152,7 +153,8 @@ void aix::Linker::ConstructJob(Compilation &C, const JobAction &JA,
}

const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
C.addCommand(std::make_unique<Command>(JA, *this, Exec, CmdArgs, Inputs));
C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
Exec, CmdArgs, Inputs));
}

/// AIX - AIX tool chain which can call as(1) and ld(1) directly.
Expand Down

0 comments on commit 4772b99

Please sign in to comment.