heredoc

Makefile

@@ -9,3 +9,9 @@ test:
 test-conformance:
 	go test -v -tags conformance -timeout 45m ./dockerclient
 .PHONY: test-conformance
+
+.PHONY: vendor
+vendor:
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
+	GO111MODULE=on go mod verify

builder.go

@@ -33,6 +33,11 @@ type Copy struct {
 	Chmod string
 }
 
+type File struct {
+	Name  string
+	Data  string
+}
+
 // Run defines a run operation required in the container.
 type Run struct {
 	Shell bool
@@ -41,6 +46,7 @@ type Run struct {
 	Mounts []string
 	// Network specifies the network mode to run the container with
 	Network string
+	Files []File
 }
 
 type Executor interface {

dockerfile/parser/parser.go

@@ -15,6 +15,8 @@ import (
 
 	sRegexp "github.com/containers/storage/pkg/regexp"
 	"github.com/containers/storage/pkg/system"
+	buildkitparser "github.com/moby/buildkit/frontend/dockerfile/parser"
+	buildkitshell "github.com/moby/buildkit/frontend/dockerfile/shell"
 	"github.com/openshift/imagebuilder/dockerfile/command"
 )
 
@@ -30,14 +32,15 @@ import (
 // but lucky for us the Dockerfile isn't very complicated. This structure
 // works a little more effectively than a "proper" parse tree for our needs.
 type Node struct {
-	Value      string          // actual content
-	Next       *Node           // the next item in the current sexp
-	Children   []*Node         // the children of this sexp
-	Attributes map[string]bool // special attributes for this node
-	Original   string          // original line used before parsing
-	Flags      []string        // only top Node should have this set
-	StartLine  int             // the line in the original dockerfile where the node begins
-	EndLine    int             // the line in the original dockerfile where the node ends
+	Value      string                   // actual content
+	Next       *Node                    // the next item in the current sexp
+	Children   []*Node                  // the children of this sexp
+	Heredocs   []buildkitparser.Heredoc // extra heredoc content attachments
+	Attributes map[string]bool          // special attributes for this node
+	Original   string                   // original line used before parsing
+	Flags      []string                 // only top Node should have this set
+	StartLine  int                      // the line in the original dockerfile where the node begins
+	EndLine    int                      // the line in the original dockerfile where the node ends
 }
 
 // Dump dumps the AST defined by `node` as a list of sexps.
@@ -70,6 +73,24 @@ func (node *Node) lines(start, end int) {
 	node.EndLine = end
 }
 
+func (node *Node) canContainHeredoc() bool {
+	// check for compound commands, like ONBUILD
+	if ok := heredocCompoundDirectives[strings.ToLower(node.Value)]; ok {
+		if node.Next != nil && len(node.Next.Children) > 0 {
+			node = node.Next.Children[0]
+		}
+	}
+
+	if ok := heredocDirectives[strings.ToLower(node.Value)]; !ok {
+		return false
+	}
+	if isJSON := node.Attributes["json"]; isJSON {
+		return false
+	}
+
+	return true
+}
+
 // AddChild adds a new child node, and updates line information
 func (node *Node) AddChild(child *Node, startLine, endLine int) {
 	child.lines(startLine, endLine)
@@ -85,6 +106,7 @@ var (
 	tokenWhitespace      = sRegexp.Delayed(`[\t\v\f\r ]+`)
 	tokenEscapeCommand   = sRegexp.Delayed(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
 	tokenPlatformCommand = sRegexp.Delayed(`^#[ \t]*platform[ \t]*=[ \t]*(?P<platform>.*)$`)
+	reHeredoc            = regexp.MustCompile(`^(\d*)<<(-?)([^<]*)$`)
 	tokenComment         = sRegexp.Delayed(`^#.*$`)
 )
 
@@ -94,6 +116,20 @@ const DefaultEscapeToken = '\\'
 // defaultPlatformToken is the platform assumed for the build if not explicitly provided
 var defaultPlatformToken = runtime.GOOS
 
+var (
+	// Directives allowed to contain heredocs
+	heredocDirectives = map[string]bool{
+		command.Add:  true,
+		command.Copy: true,
+		command.Run:  true,
+	}
+
+	// Directives allowed to contain directives containing heredocs
+	heredocCompoundDirectives = map[string]bool{
+		command.Onbuild: true,
+	}
+)
+
 // Directive is the structure used during a build run to hold the state of
 // parsing directives.
 type Directive struct {
@@ -309,6 +345,38 @@ func Parse(rwc io.Reader) (*Result, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		if child.canContainHeredoc() {
+			heredocs, err := heredocsFromLine(line)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, heredoc := range heredocs {
+				terminator := []byte(heredoc.Name)
+				terminated := false
+				for scanner.Scan() {
+					bytesRead := scanner.Bytes()
+					currentLine++
+
+					possibleTerminator := trimNewline(bytesRead)
+					if heredoc.Chomp {
+						possibleTerminator = trimLeadingTabs(possibleTerminator)
+					}
+					if bytes.Equal(possibleTerminator, terminator) {
+						terminated = true
+						break
+					}
+					heredoc.Content += string(bytesRead)
+				}
+				if !terminated {
+					return nil, errors.New("unterminated heredoc")
+				}
+
+				child.Heredocs = append(child.Heredocs, heredoc)
+			}
+		}
+
 		root.AddChild(child, startLine, currentLine)
 	}
 
@@ -323,6 +391,26 @@ func Parse(rwc io.Reader) (*Result, error) {
 	}, nil
 }
 
+func heredocsFromLine(line string) ([]buildkitparser.Heredoc, error) {
+	shlex := buildkitshell.NewLex('\\')
+	shlex.RawQuotes = true
+	shlex.RawEscapes = true
+	shlex.SkipUnsetEnv = true
+	words, _ := shlex.ProcessWords(line, []string{})
+
+	var docs []buildkitparser.Heredoc
+	for _, word := range words {
+		heredoc, err := buildkitparser.ParseHeredoc(word)
+		if err != nil {
+			return nil, err
+		}
+		if heredoc != nil {
+			docs = append(docs, *heredoc)
+		}
+	}
+	return docs, nil
+}
+
 func trimComments(src []byte) []byte {
 	return tokenComment.ReplaceAll(src, []byte{})
 }
@@ -331,6 +419,16 @@ func trimWhitespace(src []byte) []byte {
 	return bytes.TrimLeftFunc(src, unicode.IsSpace)
 }
 
+func trimLeadingWhitespace(src []byte) []byte {
+	return bytes.TrimLeftFunc(src, unicode.IsSpace)
+}
+func trimLeadingTabs(src []byte) []byte {
+	return bytes.TrimLeft(src, "\t")
+}
+func trimNewline(src []byte) []byte {
+	return bytes.TrimRight(src, "\r\n")
+}
+
 func isEmptyContinuationLine(line []byte) bool {
 	return len(trimComments(trimWhitespace(line))) == 0
 }

go.mod

@@ -8,14 +8,15 @@ require (
 	github.com/docker/distribution v2.8.2+incompatible
 	github.com/docker/docker v24.0.2+incompatible
 	github.com/fsouza/go-dockerclient v1.9.7
-	github.com/sirupsen/logrus v1.9.3
+	github.com/moby/buildkit v0.10.6
 	github.com/stretchr/testify v1.8.4
 	k8s.io/klog v1.0.0
 )
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
@@ -36,6 +37,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	golang.org/x/mod v0.9.0 // indirect

go.sum

@@ -11,6 +11,8 @@ github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2u
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/containerd v1.7.2 h1:UF2gdONnxO8I6byZXDi5sXWiWvlW3D/sci7dTQimEJo=
 github.com/containerd/containerd v1.7.2/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI=
+github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
+github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
 github.com/containers/storage v1.46.1 h1:GcAe8J0Y6T2CF70fXPojUpnme6zXamuzGCrNujVtIGE=
 github.com/containers/storage v1.46.1/go.mod h1:81vNDX4h+nXJ2o0D6Yqy6JGXDYJGVpHZpz0nr09iJuQ=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -60,6 +62,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/moby/buildkit v0.10.6 h1:DJlEuLIgnu34HQKF4n9Eg6q2YqQVC0eOpMb4p2eRS2w=
+github.com/moby/buildkit v0.10.6/go.mod h1:tQuuyTWtOb9D+RE425cwOCUkX0/oZ+5iBZ+uWpWQ9bU=
 github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=
 github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=