POC: Speed up compilation by freezing container during compilation. (#…

…11940)

Instead of rebuilding the entire project every time we want to compile a
single fuzz target, a better workflow is to build the project once and
somehow compile the target against the already compiled project code.
This POC does that by interrupting building in when it detects it is
compiling the fuzz target.
On detection it does the following:
1. Writes the command to /out/statefile TODO: write the cwd.
2. Commits the current container as "frozen" for use later. TODO: make
this changeable.
3. Returns 1 so compilation stops. TODO: It would be better to exit the
container.
This step may be important to prevent clean up of the environment.

Then the frozen container can be used to compile fuzz targets against
the project without recompiling the project in its entirety.

TODO:
1. Support this in oss-fuzz-gen
2. Install docker command line tool in base-builder (or use sneaky
inheritance) because it must be used within the container.
3. Automate the compilation of the new fuzz target

infra/base-images/base-builder/Dockerfile

@@ -170,8 +170,10 @@ COPY bazel_build_fuzz_tests \
 
 # TODO: Build this as part of a multi-stage build.
 ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang-jcc /usr/local/bin/
-ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang++-jcc /usr/local/bin/
-RUN chmod +x /usr/local/bin/clang-jcc && chmod +x /usr/local/bin/clang++-jcc
+ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang++-jcc /usr/local/bin
+ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang-jcc2 /usr/local/bin/
+ADD https://commondatastorage.googleapis.com/clusterfuzz-builds/jcc/clang++-jcc2 /usr/local/bin
+RUN chmod +x /usr/local/bin/clang-jcc /usr/local/bin/clang++-jcc /usr/local/bin/clang-jcc2 /usr/local/bin/clang++-jcc2
 
 COPY llvmsymbol.diff $SRC
 COPY detect_repo.py /opt/cifuzz/

infra/base-images/base-builder/jcc/build_jcc.bash

@@ -17,7 +17,9 @@
 ################################################################################
 
 go build jcc.go
-cp jcc clang
-cp jcc clang++
+go build jcc2.go
 gsutil cp jcc gs://clusterfuzz-builds/jcc/clang++-jcc
 gsutil cp jcc gs://clusterfuzz-builds/jcc/clang-jcc
+
+gsutil cp jcc2 gs://clusterfuzz-builds/jcc/clang++-jcc2
+gsutil cp jcc2 gs://clusterfuzz-builds/jcc/clang-jcc2

infra/base-images/base-builder/jcc/jcc.go

@@ -25,7 +25,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"slices"
 	"strings"
 )
 
@@ -169,63 +168,6 @@ func CorrectMissingHeaders(bin string, cmd []string) ([]string, bool, error) {
 	return cmd, false, nil
 }
 
-func EnsureDir(dirPath string) {
-	// Checks if a path is an existing directory, otherwise create one.
-	if pathInfo, err := os.Stat(dirPath); err == nil {
-		if isDir := pathInfo.IsDir(); !isDir {
-			panic(dirPath + " exists but is not a directory.")
-		}
-	} else if errors.Is(err, fs.ErrNotExist) {
-		if err := os.MkdirAll(dirPath, 0755); err != nil {
-			panic("Failed to create directory: " + dirPath + ".")
-		}
-		fmt.Println("Created directory: " + dirPath + ".")
-	} else {
-		panic("An error occurred in os.Stat(" + dirPath + "): " + err.Error())
-	}
-}
-
-func GenerateAST(bin string, args []string, filePath string) {
-	// Generates AST.
-	outFile, err := os.Create(filePath)
-	if err != nil {
-		fmt.Println(err)
-	}
-	defer outFile.Close()
-
-	cmd := exec.Command(bin, args...)
-	cmd.Stdout = outFile
-	cmd.Run()
-}
-
-func GenerateASTs(bin string, args []string, astDir string) {
-	// Generates an AST for each C/CPP file in the command.
-	// Cannot save AST when astDir is not available.
-	EnsureDir(astDir)
-
-	// Target file suffixes.
-	suffixes := []string{".cpp", ".cc", ".cxx", ".c++", ".c", ".h", ".hpp"}
-	// C/CPP targets in the command.
-	targetFiles := []string{}
-	// Flags to generate AST.
-	flags := []string{"-Xclang", "-ast-dump=json", "-fsyntax-only"}
-	for _, arg := range args {
-		targetFileExt := strings.ToLower(filepath.Ext(arg))
-		if slices.Contains(suffixes, targetFileExt) {
-			targetFiles = append(targetFiles, arg)
-			continue
-		}
-		flags = append(flags, arg)
-	}
-
-	// Generate an AST for each target file. Skips AST generation when a
-	// command has no target file (e.g., during linking).
-	for _, targetFile := range targetFiles {
-		filePath := filepath.Join(astDir, fmt.Sprintf("%s.ast", filepath.Base(targetFile)))
-		GenerateAST(bin, append(flags, targetFile), filePath)
-	}
-}
-
 func ExecBuildCommand(bin string, args []string) (int, string, string) {
 	// Executes the original command.
 	cmd := exec.Command(bin, args...)
@@ -238,10 +180,6 @@ func ExecBuildCommand(bin string, args []string) (int, string, string) {
 }
 
 func Compile(bin string, args []string) (int, string, string) {
-	// Generate ASTs f we define this ENV var.
-	if astDir := os.Getenv("JCC_GENERATE_AST_DIR"); astDir != "" {
-		GenerateASTs(bin, args, astDir)
-	}
 	// Run the actual command.
 	return ExecBuildCommand(bin, args)
 }
@@ -360,7 +298,7 @@ func WriteStdErrOut(args []string, outstr string, errstr string) {
 	fmt.Print(outstr)
 	fmt.Fprint(os.Stderr, errstr)
 	// Record what compile args produced the error and the error itself in log file.
-	AppendStringToFile("/workspace/err.log", fmt.Sprintf("%s\n", args) + errstr)
+	AppendStringToFile("/workspace/err.log", fmt.Sprintf("%s\n", args)+errstr)
 }
 
 func main() {