Moved a couple of functions in the proper builder package

arduino/builder/cpp.go

@@ -0,0 +1,74 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+// QuoteCppString returns the given string as a quoted string for use with the C
+// preprocessor. This adds double quotes around it and escapes any
+// double quotes and backslashes in the string.
+func QuoteCppString(str string) string {
+	str = strings.Replace(str, "\\", "\\\\", -1)
+	str = strings.Replace(str, "\"", "\\\"", -1)
+	return "\"" + str + "\""
+}
+
+// Parse a C-preprocessor string as emitted by the preprocessor. This
+// is a string contained in double quotes, with any backslashes or
+// quotes escaped with a backslash. If a valid string was present at the
+// start of the given line, returns the unquoted string contents, the
+// remainder of the line (everything after the closing "), and true.
+// Otherwise, returns the empty string, the entire line and false.
+func ParseCppString(line string) (string, string, bool) {
+	// For details about how these strings are output by gcc, see:
+	// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
+	// Note that the documentation suggests all non-printable
+	// characters are also escaped, but the implementation does not
+	// actually do this. See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51259
+	if len(line) < 1 || line[0] != '"' {
+		return "", line, false
+	}
+
+	i := 1
+	res := ""
+	for {
+		if i >= len(line) {
+			return "", line, false
+		}
+
+		c, width := utf8.DecodeRuneInString(line[i:])
+
+		switch c {
+		case '\\':
+			// Backslash, next character is used unmodified
+			i += width
+			if i >= len(line) {
+				return "", line, false
+			}
+			res += string(line[i])
+		case '"':
+			// Quote, end of string
+			return res, line[i+width:], true
+		default:
+			res += string(c)
+		}
+
+		i += width
+	}
+}

arduino/builder/cpp_test.go

@@ -0,0 +1,46 @@
+package builder_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseCppString(t *testing.T) {
+	_, _, ok := builder.ParseCppString(`foo`)
+	require.Equal(t, false, ok)
+
+	_, _, ok = builder.ParseCppString(`"foo`)
+	require.Equal(t, false, ok)
+
+	str, rest, ok := builder.ParseCppString(`"foo"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"foo\\bar"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo\bar`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
+	require.Equal(t, ` and "then" some`, rest)
+
+	str, rest, ok = builder.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"/home/ççç/"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = builder.ParseCppString(`"/home/ççç/ /$sdsdd\\"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
+	require.Equal(t, ``, rest)
+}

arduino/builder/sketch.go

@@ -19,7 +19,6 @@ import (
 	"bytes"
 	"fmt"
 	"regexp"
-	"strings"
 
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/i18n"
@@ -34,15 +33,6 @@ var (
 	tr               = i18n.Tr
 )
 
-// quoteCppString returns the given string as a quoted string for use with the C
-// preprocessor. This adds double quotes around it and escapes any
-// double quotes and backslashes in the string.
-func quoteCppString(str string) string {
-	str = strings.Replace(str, "\\", "\\\\", -1)
-	str = strings.Replace(str, "\"", "\\\"", -1)
-	return "\"" + str + "\""
-}
-
 // PrepareSketchBuildPath copies the sketch source files in the build path.
 // The .ino files are merged together to create a .cpp file (by the way, the
 // .cpp file still needs to be Arduino-preprocessed to compile).
@@ -106,7 +96,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
 		lineOffset++
 	}
 
-	mergedSource += "#line 1 " + quoteCppString(sk.MainFile.String()) + "\n"
+	mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.String()) + "\n"
 	mergedSource += mainSrc + "\n"
 	lineOffset++
 
@@ -115,7 +105,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
 		if err != nil {
 			return 0, "", err
 		}
-		mergedSource += "#line 1 " + quoteCppString(file.String()) + "\n"
+		mergedSource += "#line 1 " + QuoteCppString(file.String()) + "\n"
 		mergedSource += src + "\n"
 	}
 
@@ -155,7 +145,7 @@ func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, over
 		}
 
 		// tag each addtional file with the filename of the source it was copied from
-		sourceBytes = append([]byte("#line 1 "+quoteCppString(file.String())+"\n"), sourceBytes...)
+		sourceBytes = append([]byte("#line 1 "+QuoteCppString(file.String())+"\n"), sourceBytes...)
 
 		err = writeIfDifferent(sourceBytes, targetPath)
 		if err != nil {

legacy/builder/container_add_prototypes.go

@@ -16,11 +16,17 @@
 package builder
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
+	"io"
+	"strconv"
+	"strings"
 
 	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
+	"github.com/arduino/go-paths-helper"
 	"github.com/pkg/errors"
 )
 
@@ -51,7 +57,7 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 	if src, err := targetFilePath.ReadFile(); err != nil {
 		return err
 	} else {
-		ctx.SketchSourceAfterCppPreprocessing = FilterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
+		ctx.SketchSourceAfterCppPreprocessing = filterSketchSource(ctx.Sketch, bytes.NewReader(src), false)
 	}
 
 	commands := []types.Command{
@@ -73,3 +79,60 @@ func PreprocessSketchWithCtags(ctx *types.Context) error {
 
 	return nil
 }
+
+func filterSketchSource(sketch *sketch.Sketch, source io.Reader, removeLineMarkers bool) string {
+	fileNames := paths.NewPathList()
+	fileNames.Add(sketch.MainFile)
+	fileNames.AddAll(sketch.OtherSketchFiles)
+
+	inSketch := false
+	filtered := ""
+
+	scanner := bufio.NewScanner(source)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if filename := parseLineMarker(line); filename != nil {
+			inSketch = fileNames.Contains(filename)
+			if inSketch && removeLineMarkers {
+				continue
+			}
+		}
+
+		if inSketch {
+			filtered += line + "\n"
+		}
+	}
+
+	return filtered
+}
+
+// Parses the given line as a gcc line marker and returns the contained
+// filename.
+func parseLineMarker(line string) *paths.Path {
+	// A line marker contains the line number and filename and looks like:
+	// # 123 /path/to/file.cpp
+	// It can be followed by zero or more flag number that indicate the
+	// preprocessor state and can be ignored.
+	// For exact details on this format, see:
+	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
+
+	split := strings.SplitN(line, " ", 3)
+	if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' {
+		return nil
+	}
+
+	_, err := strconv.Atoi(split[1])
+	if err != nil {
+		return nil
+	}
+
+	// If we get here, we found a # followed by a line number, so
+	// assume this is a line marker and see if the rest of the line
+	// starts with a string containing the filename
+	str, rest, ok := bldr.ParseCppString(split[2])
+
+	if ok && (rest == "" || rest[0] == ' ') {
+		return paths.New(str)
+	}
+	return nil
+}

legacy/builder/create_cmake_rule.go

@@ -120,7 +120,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error {
 	}
 
 	// Use old ctags method to generate export file
-	ctx.SketchSourceMerged = FilterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true)
+	ctx.SketchSourceMerged = filterSketchSource(ctx.Sketch, strings.NewReader(ctx.SketchSourceMerged), true)
 
 	err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions)
 	if err != nil {

legacy/builder/test/utils_test.go

@@ -65,41 +65,3 @@ func TestQuoteCppString(t *testing.T) {
 		require.Equal(t, expected, utils.QuoteCppString(input))
 	}
 }
-
-func TestParseCppString(t *testing.T) {
-	_, _, ok := utils.ParseCppString(`foo`)
-	require.Equal(t, false, ok)
-
-	_, _, ok = utils.ParseCppString(`"foo`)
-	require.Equal(t, false, ok)
-
-	str, rest, ok := utils.ParseCppString(`"foo"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `foo`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"foo\\bar"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `foo\bar`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
-	require.Equal(t, ` and "then" some`, rest)
-
-	str, rest, ok = utils.ParseCppString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"/home/ççç/"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `/home/ççç/`, str)
-	require.Equal(t, ``, rest)
-
-	str, rest, ok = utils.ParseCppString(`"/home/ççç/ /$sdsdd\\"`)
-	require.Equal(t, true, ok)
-	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
-	require.Equal(t, ``, rest)
-}