Skip to content

Commit

Permalink
Add support for compile_command.json output (#1081)
Browse files Browse the repository at this point in the history
* Draft: Support generating a compile_commands.json file

This is still very rough and unfinished.

* Added compile flag to produce only the compilation database

* Save compilation database inside build path

* Removed 'Writing compilation database...' message

Also slightly refactore code

* Added missing (c) header

* Renamed some functions, did some small cleanups

* compilation database: made some fields public

* compilation database: added LoadCompilationDatabase method

* Added unit tests for compilation_database

Co-authored-by: Matthijs Kooijman <matthijs@stdin.nl>
  • Loading branch information
cmaglie and matthijskooijman committed Dec 15, 2020
1 parent bf7a319 commit 441f8eb
Show file tree
Hide file tree
Showing 16 changed files with 303 additions and 97 deletions.
95 changes: 95 additions & 0 deletions arduino/builder/compilation_database.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to license@arduino.cc.

package builder

import (
"encoding/json"
"fmt"
"os"
"os/exec"

"github.com/arduino/go-paths-helper"
)

// CompilationDatabase keeps track of all the compile commands run by the builder
type CompilationDatabase struct {
Contents []CompilationCommand
File *paths.Path
}

// CompilationCommand keeps track of a single run of a compile command
type CompilationCommand struct {
Directory string `json:"directory"`
Command string `json:"command,omitempty"`
Arguments []string `json:"arguments,omitempty"`
File string `json:"file"`
}

// NewCompilationDatabase creates an empty CompilationDatabase
func NewCompilationDatabase(filename *paths.Path) *CompilationDatabase {
return &CompilationDatabase{
File: filename,
}
}

// LoadCompilationDatabase reads a compilation database from a file
func LoadCompilationDatabase(file *paths.Path) (*CompilationDatabase, error) {
f, err := file.ReadFile()
if err != nil {
return nil, err
}
res := &CompilationDatabase{
File: file,
Contents: []CompilationCommand{},
}
return res, json.Unmarshal(f, &res.Contents)
}

// SaveToFile save the CompilationDatabase to file as a clangd-compatible compile_commands.json,
// see https://clang.llvm.org/docs/JSONCompilationDatabase.html
func (db *CompilationDatabase) SaveToFile() {
if jsonContents, err := json.MarshalIndent(db.Contents, "", " "); err != nil {
fmt.Printf("Error serializing compilation database: %s", err)
return
} else if err := db.File.WriteFile(jsonContents); err != nil {
fmt.Printf("Error writing compilation database: %s", err)
}
}

func dirForCommand(command *exec.Cmd) string {
// This mimics what Cmd.Run also does: Use Dir if specified,
// current directory otherwise
if command.Dir != "" {
return command.Dir
}
dir, err := os.Getwd()
if err != nil {
fmt.Printf("Error getting current directory for compilation database: %s", err)
return ""
}
return dir
}

// Add adds a new CompilationDatabase entry
func (db *CompilationDatabase) Add(target *paths.Path, command *exec.Cmd) {
entry := CompilationCommand{
Directory: dirForCommand(command),
Arguments: command.Args,
File: target.String(),
}

db.Contents = append(db.Contents, entry)
}
46 changes: 46 additions & 0 deletions arduino/builder/compilation_database_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// This file is part of arduino-cli.
//
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to license@arduino.cc.

package builder

import (
"os/exec"
"testing"

"github.com/arduino/go-paths-helper"
"github.com/stretchr/testify/require"
)

func TestCompilationDatabase(t *testing.T) {
tmpfile, err := paths.WriteToTempFile([]byte{}, nil, "")
require.NoError(t, err)
defer tmpfile.Remove()

cmd := exec.Command("gcc", "arg1", "arg2")
db := NewCompilationDatabase(tmpfile)
db.Add(paths.New("test"), cmd)
db.SaveToFile()

db2, err := LoadCompilationDatabase(tmpfile)
require.NoError(t, err)
require.Equal(t, db, db2)
require.Len(t, db2.Contents, 1)
require.Equal(t, db2.Contents[0].File, "test")
require.Equal(t, db2.Contents[0].Command, "")
require.Equal(t, db2.Contents[0].Arguments, []string{"gcc", "arg1", "arg2"})
cwd, err := paths.Getwd()
require.NoError(t, err)
require.Equal(t, db2.Contents[0].Directory, cwd.String())
}
75 changes: 39 additions & 36 deletions cli/compile/compile.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,25 +35,26 @@ import (
)

var (
fqbn string // Fully Qualified Board Name, e.g.: arduino:avr:uno.
showProperties bool // Show all build preferences used instead of compiling.
preprocess bool // Print preprocessed code to stdout.
buildCachePath string // Builds of 'core.a' are saved into this path to be cached and reused.
buildPath string // Path where to save compiled files.
buildProperties []string // List of custom build properties separated by commas. Or can be used multiple times for multiple properties.
warnings string // Used to tell gcc which warning level to use.
verbose bool // Turns on verbose mode.
quiet bool // Suppresses almost every output.
vidPid string // VID/PID specific build properties.
uploadAfterCompile bool // Upload the binary after the compilation.
port string // Upload port, e.g.: COM10 or /dev/ttyACM0.
verify bool // Upload, verify uploaded binary after the upload.
exportDir string // The compiled binary is written to this file
libraries []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
optimizeForDebug bool // Optimize compile output for debug, not for release
programmer string // Use the specified programmer to upload
clean bool // Cleanup the build folder and do not use any cached build
exportBinaries bool // Copies compiled binaries to sketch folder when true
fqbn string // Fully Qualified Board Name, e.g.: arduino:avr:uno.
showProperties bool // Show all build preferences used instead of compiling.
preprocess bool // Print preprocessed code to stdout.
buildCachePath string // Builds of 'core.a' are saved into this path to be cached and reused.
buildPath string // Path where to save compiled files.
buildProperties []string // List of custom build properties separated by commas. Or can be used multiple times for multiple properties.
warnings string // Used to tell gcc which warning level to use.
verbose bool // Turns on verbose mode.
quiet bool // Suppresses almost every output.
vidPid string // VID/PID specific build properties.
uploadAfterCompile bool // Upload the binary after the compilation.
port string // Upload port, e.g.: COM10 or /dev/ttyACM0.
verify bool // Upload, verify uploaded binary after the upload.
exportDir string // The compiled binary is written to this file
libraries []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.
optimizeForDebug bool // Optimize compile output for debug, not for release
programmer string // Use the specified programmer to upload
clean bool // Cleanup the build folder and do not use any cached build
exportBinaries bool // Copies compiled binaries to sketch folder when true
compilationDatabaseOnly bool // Only create compilation database without actually compiling
)

// NewCommand created a new `compile` command
Expand Down Expand Up @@ -94,6 +95,7 @@ func NewCommand() *cobra.Command {
"List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.")
command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.")
command.Flags().StringVarP(&programmer, "programmer", "P", "", "Optional, use the specified programmer to upload.")
command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, "Just produce the compilation database, without actually compiling.")
command.Flags().BoolVar(&clean, "clean", false, "Optional, cleanup the build folder and do not use any cached build.")
// We must use the following syntax for this flag since it's also bound to settings, we could use the other one too
// but it wouldn't make sense since we still must explicitly set the exportBinaries variable by reading from settings.
Expand Down Expand Up @@ -127,23 +129,24 @@ func run(cmd *cobra.Command, args []string) {
exportBinaries = configuration.Settings.GetBool("sketch.always_export_binaries")

compileReq := &rpc.CompileReq{
Instance: inst,
Fqbn: fqbn,
SketchPath: sketchPath.String(),
ShowProperties: showProperties,
Preprocess: preprocess,
BuildCachePath: buildCachePath,
BuildPath: buildPath,
BuildProperties: buildProperties,
Warnings: warnings,
Verbose: verbose,
Quiet: quiet,
VidPid: vidPid,
ExportDir: exportDir,
Libraries: libraries,
OptimizeForDebug: optimizeForDebug,
Clean: clean,
ExportBinaries: exportBinaries,
Instance: inst,
Fqbn: fqbn,
SketchPath: sketchPath.String(),
ShowProperties: showProperties,
Preprocess: preprocess,
BuildCachePath: buildCachePath,
BuildPath: buildPath,
BuildProperties: buildProperties,
Warnings: warnings,
Verbose: verbose,
Quiet: quiet,
VidPid: vidPid,
ExportDir: exportDir,
Libraries: libraries,
OptimizeForDebug: optimizeForDebug,
Clean: clean,
ExportBinaries: exportBinaries,
CreateCompilationDatabaseOnly: compilationDatabaseOnly,
}
compileOut := new(bytes.Buffer)
compileErr := new(bytes.Buffer)
Expand Down
5 changes: 4 additions & 1 deletion commands/compile/compile.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
} else {
builderCtx.BuildPath = paths.New(req.GetBuildPath())
}

if err = builderCtx.BuildPath.MkdirAll(); err != nil {
return nil, fmt.Errorf("cannot create build directory: %s", err)
}
builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
builderCtx.BuildPath.Join("compile_commands.json"),
)

builderCtx.Verbose = req.GetVerbose()

Expand Down Expand Up @@ -188,6 +190,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W
builderCtx.ExecStderr = errStream
builderCtx.SetLogger(i18n.LoggerToCustomStreams{Stdout: outStream, Stderr: errStream})
builderCtx.Clean = req.GetClean()
builderCtx.OnlyUpdateCompilationDatabase = req.GetCreateCompilationDatabaseOnly()

// Use defer() to create an rpc.CompileResp with all the information available at the
// moment of return.
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.14

require (
github.com/arduino/board-discovery v0.0.0-20180823133458-1ba29327fb0c
github.com/arduino/go-paths-helper v1.3.2
github.com/arduino/go-paths-helper v1.4.0
github.com/arduino/go-properties-orderedmap v1.3.0
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b
github.com/arduino/go-win32-utils v0.0.0-20180330194947-ed041402e83b
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3
github.com/arduino/go-paths-helper v1.2.0/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
github.com/arduino/go-paths-helper v1.3.2 h1:5U9TSKQODiwSVgTxskC0PNl0l0Vf40GUlp99Zy2SK8w=
github.com/arduino/go-paths-helper v1.3.2/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
github.com/arduino/go-paths-helper v1.4.0 h1:ilnseAdxmN1bFnLxxXHRtcdmt9jBf3O4jtYfWfqule4=
github.com/arduino/go-paths-helper v1.4.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
github.com/arduino/go-properties-orderedmap v1.3.0 h1:4No/vQopB36e7WUIk6H6TxiSEJPiMrVOCZylYmua39o=
github.com/arduino/go-properties-orderedmap v1.3.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
github.com/arduino/go-timeutils v0.0.0-20171220113728-d1dd9e313b1b h1:9hDi4F2st6dbLC3y4i02zFT5quS4X6iioWifGlVwfy4=
Expand Down
4 changes: 4 additions & 0 deletions legacy/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,10 @@ func (s *Builder) Run(ctx *types.Context) error {

mainErr := runCommands(ctx, commands)

if ctx.CompilationDatabase != nil {
ctx.CompilationDatabase.SaveToFile()
}

commands = []types.Command{
&PrintUsedAndNotUsedLibraries{SketchError: mainErr != nil},

Expand Down
29 changes: 20 additions & 9 deletions legacy/builder/builder_utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,18 +247,24 @@ func compileFileWithRecipe(ctx *types.Context, sourcePath *paths.Path, source *p
if err != nil {
return nil, errors.WithStack(err)
}
if !objIsUpToDate {
command, err := PrepareCommandForRecipe(properties, recipe, false)
if err != nil {
return nil, errors.WithStack(err)
}

command, err := PrepareCommandForRecipe(properties, recipe, false)
if err != nil {
return nil, errors.WithStack(err)
}
if ctx.CompilationDatabase != nil {
ctx.CompilationDatabase.Add(source, command)
}
if !objIsUpToDate && !ctx.OnlyUpdateCompilationDatabase {
_, _, err = utils.ExecCommand(ctx, command, utils.ShowIfVerbose /* stdout */, utils.Show /* stderr */)
if err != nil {
return nil, errors.WithStack(err)
}
} else if ctx.Verbose {
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, objectFile)
if objIsUpToDate {
logger.Println(constants.LOG_LEVEL_INFO, constants.MSG_USING_PREVIOUS_COMPILED_FILE, objectFile)
} else {
logger.Println("info", "Skipping compile of: {0}", objectFile)
}
}

return objectFile, nil
Expand Down Expand Up @@ -452,10 +458,15 @@ func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile
logger := ctx.GetLogger()
archiveFilePath := buildPath.JoinPath(archiveFile)

rebuildArchive := false
if ctx.OnlyUpdateCompilationDatabase {
if ctx.Verbose {
logger.Println("info", "Skipping archive creation of: {0}", archiveFilePath)
}
return archiveFilePath, nil
}

if archiveFileStat, err := archiveFilePath.Stat(); err == nil {

rebuildArchive := false
for _, objectFile := range objectFilesToArchive {
objectFileStat, err := objectFile.Stat()
if err != nil || objectFileStat.ModTime().After(archiveFileStat.ModTime()) {
Expand Down
4 changes: 4 additions & 0 deletions legacy/builder/merge_sketch_with_bootloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ import (
type MergeSketchWithBootloader struct{}

func (s *MergeSketchWithBootloader) Run(ctx *types.Context) error {
if ctx.OnlyUpdateCompilationDatabase {
return nil
}

buildProperties := ctx.BuildProperties
if !buildProperties.ContainsKey(constants.BUILD_PROPERTIES_BOOTLOADER_NOBLINK) && !buildProperties.ContainsKey(constants.BUILD_PROPERTIES_BOOTLOADER_FILE) {
return nil
Expand Down
6 changes: 4 additions & 2 deletions legacy/builder/phases/core_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,9 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
archivedCoreName := GetCachedCoreArchiveFileName(buildProperties.Get(constants.BUILD_PROPERTIES_FQBN),
buildProperties.Get("compiler.optimization_flags"), realCoreFolder)
targetArchivedCore = buildCachePath.Join(archivedCoreName)
canUseArchivedCore := !ctx.Clean && !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)
canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase &&
!ctx.Clean &&
!builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore)

if canUseArchivedCore {
// use archived core
Expand All @@ -116,7 +118,7 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path
}

// archive core.a
if targetArchivedCore != nil {
if targetArchivedCore != nil && !ctx.OnlyUpdateCompilationDatabase {
err := archiveFile.CopyTo(targetArchivedCore)
if ctx.Verbose {
if err == nil {
Expand Down
7 changes: 7 additions & 0 deletions legacy/builder/phases/linker.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ import (
type Linker struct{}

func (s *Linker) Run(ctx *types.Context) error {
if ctx.OnlyUpdateCompilationDatabase {
if ctx.Verbose {
ctx.GetLogger().Println("info", "Skip linking of final executable.")
}
return nil
}

objectFilesSketch := ctx.SketchObjectFiles
objectFilesLibraries := ctx.LibrariesObjectFiles
objectFilesCore := ctx.CoreObjectsFiles
Expand Down
4 changes: 3 additions & 1 deletion legacy/builder/phases/sizer.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ type Sizer struct {
}

func (s *Sizer) Run(ctx *types.Context) error {

if ctx.OnlyUpdateCompilationDatabase {
return nil
}
if s.SketchError {
return nil
}
Expand Down

0 comments on commit 441f8eb

Please sign in to comment.