Skip to content

Commit

Permalink
Report compiler errors in machine readable format (#2182)
Browse files Browse the repository at this point in the history
* Added gRPC protocol for compiler diagnostics reporting

* Builder now support parsing of compiler output

* Added gcc output parser

* Implementation of compile output parser in gRPC command

* Tell prettier to ignore testdata files

* Added proper result structure for diagnostics

* Added integration test

* Fixed parser bug and added unit test
  • Loading branch information
cmaglie committed Nov 8, 2023
1 parent 7a9be52 commit 0e5f629
Show file tree
Hide file tree
Showing 23 changed files with 1,646 additions and 47 deletions.
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
.vs/
.ionide/

testdata
/arduino/libraries/librariesindex/testdata/invalid.json
/arduino/security/testdata/
/.licenses/
Expand Down
36 changes: 34 additions & 2 deletions arduino/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/arduino/arduino-cli/arduino/builder/internal/compilation"
"github.com/arduino/arduino-cli/arduino/builder/internal/detector"
"github.com/arduino/arduino-cli/arduino/builder/internal/diagnostics"
"github.com/arduino/arduino-cli/arduino/builder/internal/logger"
"github.com/arduino/arduino-cli/arduino/builder/internal/progress"
"github.com/arduino/arduino-cli/arduino/builder/internal/utils"
Expand All @@ -36,6 +37,7 @@ import (
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
"github.com/arduino/go-paths-helper"
"github.com/arduino/go-properties-orderedmap"
"github.com/sirupsen/logrus"
)

// ErrSketchCannotBeLocatedInBuildPath fixdoc
Expand Down Expand Up @@ -90,6 +92,12 @@ type Builder struct {
buildOptions *buildOptions

libsDetector *detector.SketchLibrariesDetector

// This is a function used to parse the output of the compiler
// It is used to extract errors and warnings
compilerOutputParser diagnostics.CompilerOutputParserCB
// and here are the diagnostics parsed from the compiler
compilerDiagnostics diagnostics.Diagnostics
}

// buildArtifacts contains the result of various build
Expand Down Expand Up @@ -189,7 +197,7 @@ func NewBuilder(
logger.Warn(string(verboseOut))
}

return &Builder{
b := &Builder{
sketch: sk,
buildProperties: buildProperties,
buildPath: buildPath,
Expand Down Expand Up @@ -226,7 +234,26 @@ func NewBuilder(
buildProperties.GetPath("runtime.platform.path"),
buildProperties.GetPath("build.core.path"), // TODO can we buildCorePath ?
),
}, nil
}

b.compilerOutputParser = func(cmdline []string, out []byte) {
compiler := diagnostics.DetectCompilerFromCommandLine(
cmdline,
false, // at the moment compiler-probing is not required
)
if compiler == nil {
logrus.Warnf("Could not detect compiler from: %s", cmdline)
return
}
diags, err := diagnostics.ParseCompilerOutput(compiler, out)
if err != nil {
logrus.Warnf("Error parsing compiler output: %s", err)
return
}
b.compilerDiagnostics = append(b.compilerDiagnostics, diags...)
}

return b, nil
}

// GetBuildProperties returns the build properties for running this build
Expand All @@ -249,6 +276,11 @@ func (b *Builder) ImportedLibraries() libraries.List {
return b.libsDetector.ImportedLibraries()
}

// CompilerDiagnostics returns the parsed compiler diagnostics
func (b *Builder) CompilerDiagnostics() diagnostics.Diagnostics {
return b.compilerDiagnostics
}

// Preprocess fixdoc
func (b *Builder) Preprocess() ([]byte, error) {
b.Progress.AddSubSteps(6)
Expand Down
6 changes: 6 additions & 0 deletions arduino/builder/compilation.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,12 @@ func (b *Builder) compileFileWithRecipe(
}
b.logger.WriteStderr(commandStderr.Bytes())

// Parse the output of the compiler to gather errors and warnings...
if b.compilerOutputParser != nil {
b.compilerOutputParser(command.GetArgs(), commandStdout.Bytes())
b.compilerOutputParser(command.GetArgs(), commandStderr.Bytes())
}

// ...and then return the error
if err != nil {
return nil, errors.WithStack(err)
Expand Down
85 changes: 85 additions & 0 deletions arduino/builder/internal/diagnostics/compiler_detection.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// This file is part of arduino-cli.
//
// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to license@arduino.cc.

package diagnostics

import (
"bytes"
"path/filepath"
"strings"

"github.com/arduino/arduino-cli/executils"
semver "go.bug.st/relaxed-semver"
)

// DetectedCompiler represents a compiler detected from a given command line
type DetectedCompiler struct {
Name string
Family string
Version *semver.Version
DetailedVersion []string
}

// This function is overridden for mocking unit tests
var runProcess = func(args ...string) []string {
if cmd, err := executils.NewProcess(nil, args...); err == nil {
out := &bytes.Buffer{}
cmd.RedirectStdoutTo(out)
cmd.Run()
return splitLines(out.Bytes())
}
return nil
}

// DetectCompilerFromCommandLine tries to detect a compiler from a given command line.
// If probeCompiler is true, the compiler may be executed with different flags to
// infer the version or capabilities.
func DetectCompilerFromCommandLine(args []string, probeCompiler bool) *DetectedCompiler {
if len(args) == 0 {
return nil
}
basename := filepath.Base(args[0])
family := ""
if strings.Contains(basename, "g++") || strings.Contains(basename, "gcc") {
family = "gcc"
}
res := &DetectedCompiler{
Name: basename,
Family: family,
}

if family == "gcc" && probeCompiler {
// Run "gcc --version" to obtain more info
res.DetailedVersion = runProcess(args[0], "--version")

// Usually on the first line we get the compiler architecture and
// version (as last field), followed by the compiler license, for
// example:
//
// g++ (Ubuntu 12.2.0-3ubuntu1) 12.2.0
// Copyright (C) 2022 Free Software Foundation, Inc.
// This is free software; see the source for copying conditions. There is NO
// warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
if len(res.DetailedVersion) > 0 {
split := strings.Split(res.DetailedVersion[0], " ")
if len(split) >= 3 {
res.Name = split[0]
res.Version, _ = semver.Parse(split[len(split)-1])
}
}
}
return res
}
79 changes: 79 additions & 0 deletions arduino/builder/internal/diagnostics/compiler_detection_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// This file is part of arduino-cli.
//
// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
//
// This software is released under the GNU General Public License version 3,
// which covers the main part of arduino-cli.
// The terms of this license can be found at:
// https://www.gnu.org/licenses/gpl-3.0.en.html
//
// You can be released from the requirements of the above licenses by purchasing
// a commercial license. Buying such a license is mandatory if you want to
// modify or otherwise use the software for commercial activities involving the
// Arduino software without disclosing the source code of your own applications.
// To purchase a commercial license, send an email to license@arduino.cc.

package diagnostics

import (
"strings"
"testing"

"github.com/stretchr/testify/require"
)

func init() {
runProcess = mockedRunProcessToGetCompilerVersion
}

func mockedRunProcessToGetCompilerVersion(args ...string) []string {
if strings.HasSuffix(args[0], "7.3.0-atmel3.6.1-arduino7/bin/avr-g++") && args[1] == "--version" {
return []string{
"avr-g++ (GCC) 7.3.0",
"Copyright (C) 2017 Free Software Foundation, Inc.",
"This is free software; see the source for copying conditions. There is NO",
"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.",
"",
}
}
if strings.HasSuffix(args[0], "7.3.0-atmel3.6.1-arduino7/bin/avr-gcc") && args[1] == "--version" {
return []string{
"avr-gcc (GCC) 7.3.0",
"Copyright (C) 2017 Free Software Foundation, Inc.",
"This is free software; see the source for copying conditions. There is NO",
"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.",
"",
}
}
if strings.HasSuffix(args[0], "xtensa-esp32-elf-gcc/gcc8_4_0-esp-2021r2-patch3/bin/xtensa-esp32-elf-g++") && args[1] == "--version" {
return []string{
"xtensa-esp32-elf-g++ (crosstool-NG esp-2021r2-patch3) 8.4.0",
"Copyright (C) 2018 Free Software Foundation, Inc.",
"This is free software; see the source for copying conditions. There is NO",
"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.",
"",
}
}

panic("missing mock for command line: " + strings.Join(args, " "))
}

func TestCompilerDetection(t *testing.T) {
comp := DetectCompilerFromCommandLine([]string{"~/.arduino15/packages/arduino/tools/avr-gcc/7.3.0-atmel3.6.1-arduino7/bin/avr-g++"}, true)
require.NotNil(t, comp)
require.Equal(t, "gcc", comp.Family)
require.Equal(t, "avr-g++", comp.Name)
require.Equal(t, "7.3.0", comp.Version.String())

comp = DetectCompilerFromCommandLine([]string{"~/.arduino15/packages/arduino/tools/avr-gcc/7.3.0-atmel3.6.1-arduino7/bin/avr-gcc"}, true)
require.NotNil(t, comp)
require.Equal(t, "gcc", comp.Family)
require.Equal(t, "avr-gcc", comp.Name)
require.Equal(t, "7.3.0", comp.Version.String())

comp = DetectCompilerFromCommandLine([]string{"/home/megabug/.arduino15/packages/esp32/tools/xtensa-esp32-elf-gcc/gcc8_4_0-esp-2021r2-patch3/bin/xtensa-esp32-elf-g++"}, true)
require.NotNil(t, comp)
require.Equal(t, "gcc", comp.Family)
require.Equal(t, "xtensa-esp32-elf-g++", comp.Name)
require.Equal(t, "8.4.0", comp.Version.String())
}

0 comments on commit 0e5f629

Please sign in to comment.