diff --git a/go.mod b/go.mod index e412db87d9a..eb3a7115b63 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,3 @@ require ( gopkg.in/ini.v1 v1.63.2 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) - -replace github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55 - -replace github.com/docker/docker => github.com/docker/docker v1.4.2-0.20200227233006-38f52c9fec82 diff --git a/go.sum b/go.sum index fa1cc229fdf..1576c97746e 100644 --- a/go.sum +++ b/go.sum @@ -203,7 +203,11 @@ github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/containerd v1.3.1-0.20200227195959-4d242818bf55/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= @@ -259,8 +263,13 @@ github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible/ github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.6.0-rc.1.0.20180327202408-83389a148052+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v1.4.2-0.20200227233006-38f52c9fec82 h1:kZwwJwYnVWtU/byBNjD9rEGWVMvwnfiKu9lFJXjrk04= -github.com/docker/docker v1.4.2-0.20200227233006-38f52c9fec82/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v17.12.0-ce-rc1.0.20200730172259-9f28837c1d93+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible h1:J2OhsbfqoBRRT048iD/tqXBvEQWQATQ8vew6LqQmDSU= +github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= diff --git a/internal/pkg/cli/svc_init.go b/internal/pkg/cli/svc_init.go index bd26f5958a2..c8714dbc57f 100644 --- a/internal/pkg/cli/svc_init.go +++ b/internal/pkg/cli/svc_init.go @@ -252,7 +252,7 @@ func (o *initSvcOpts) Execute() error { if o.dockerfilePath != "" { hc, err = parseHealthCheck(o.dockerfile(o.dockerfilePath)) if err != nil { - return fmt.Errorf("parse dockerfile %s: %w", o.dockerfilePath, err) + log.Warningf("Cannot parse the HEALTHCHECK instruction from the Dockerfile: %v\n", err) } } diff --git a/internal/pkg/docker/dockerfile/dockerfile.go b/internal/pkg/docker/dockerfile/dockerfile.go index 44c55bbf8a9..b49dab91e42 100644 --- a/internal/pkg/docker/dockerfile/dockerfile.go +++ b/internal/pkg/docker/dockerfile/dockerfile.go @@ -14,8 +14,6 @@ import ( "strings" "time" - "github.com/moby/buildkit/frontend/dockerfile/instructions" - "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/spf13/afero" ) @@ -41,7 +39,6 @@ const ( hcRetriesFlag = "retries" retriesDefault = 2 - hcInstrStartIndex = len("HEALTHCHECK ") cmdInstructionPrefix = "CMD " cmdShell = "CMD-SHELL" ) @@ -142,7 +139,7 @@ func (df *Dockerfile) parse() error { return fmt.Errorf("read Dockerfile %s error: %w", f, err) } - parsedDockerfile, err := parse(string(f)) + parsedDockerfile, err := parse(file.Name(), string(f)) if err != nil { return err } @@ -154,39 +151,29 @@ func (df *Dockerfile) parse() error { } // parse parses the contents of a Dockerfile into a Dockerfile struct. -func parse(content string) (*Dockerfile, error) { +func parse(name, content string) (*Dockerfile, error) { var df Dockerfile df.exposedPorts = []Port{} - ast, err := parser.Parse(strings.NewReader(content)) - if err != nil { - return nil, fmt.Errorf("parse reader: %w", err) - } - - for _, child := range ast.AST.Children { - // ParseInstruction converts an AST to a typed instruction. - // Does prevalidation checks before parsing - // Example of an instruction is HEALTHCHECK CMD curl -f http://localhost/ || exit 1. - instruction, err := instructions.ParseInstruction(child) - if err != nil { - return nil, fmt.Errorf("parse instructions: %w", err) - } - inst := fmt.Sprint(instruction) - - // Getting the value at a children will return the Dockerfile directive - switch d := child.Value; d { - case "expose": - currentPorts := parseExpose(inst) + lexer := lex(strings.NewReader(content)) + for { + instr := lexer.next() + switch instr.name { + case instrErr: + return nil, fmt.Errorf("scan Dockerfile %s: %s", name, instr.args) + case instrEOF: + return &df, nil + case instrExpose: + currentPorts := parseExpose(instr.args) df.exposedPorts = append(df.exposedPorts, currentPorts...) - case "healthcheck": - healthcheckOptions, err := parseHealthCheck(inst) + case instrHealthCheck: + hc, err := parseHealthCheck(instr.args) if err != nil { return nil, err } - df.healthCheck = healthcheckOptions + df.healthCheck = hc } } - return &df, nil } func parseExpose(line string) []Port { @@ -245,24 +232,31 @@ func parseExpose(line string) []Port { // parseHealthCheck takes a HEALTHCHECK directives and turns into a healthCheck struct. func parseHealthCheck(content string) (*HealthCheck, error) { - if strings.TrimSpace(content[hcInstrStartIndex:]) == "NONE" { + if strings.ToUpper(strings.TrimSpace(content)) == "NONE" { return nil, nil } if !strings.Contains(content, "CMD") { - return nil, errors.New("HEALTHCHECK instruction must contain either CMD or NONE") + return nil, errors.New("parse HEALTHCHECK: instruction must contain either CMD or NONE") } var retries int var interval, timeout, startPeriod time.Duration fs := flag.NewFlagSet("flags", flag.ContinueOnError) - fs.DurationVar(&interval, intervalFlag, intervalDefault, "") fs.DurationVar(&timeout, timeoutFlag, timeoutDefault, "") fs.DurationVar(&startPeriod, startPeriodFlag, startPeriodDefault, "") fs.IntVar(&retries, hcRetriesFlag, retriesDefault, "") - if err := fs.Parse(strings.Split(content[hcInstrStartIndex:], " ")); err != nil { - return nil, err + var instrArgs []string + for _, arg := range strings.Split(content, " ") { + if arg == "" { + continue + } + instrArgs = append(instrArgs, strings.TrimSpace(arg)) + } + + if err := fs.Parse(instrArgs); err != nil { + return nil, fmt.Errorf("parse HEALTHCHECK: %w", err) } // if HEALTHCHECK instruction is not "NONE", there must be a "CMD" instruction otherwise will error out. diff --git a/internal/pkg/docker/dockerfile/dockerfile_test.go b/internal/pkg/docker/dockerfile/dockerfile_test.go index 707afcb554f..c62d94e2cf3 100644 --- a/internal/pkg/docker/dockerfile/dockerfile_test.go +++ b/internal/pkg/docker/dockerfile/dockerfile_test.go @@ -5,6 +5,7 @@ package dockerfile import ( "bytes" + "errors" "fmt" "testing" "time" @@ -86,7 +87,7 @@ FROM nginx EXPOSE $arg `), wantedPorts: nil, - wantedErr: ErrInvalidPort{Match: "EXPOSE $arg"}, + wantedErr: ErrInvalidPort{Match: "$arg"}, }, "bad expose token multiple ports": { dockerfilePath: wantedPath, @@ -96,7 +97,7 @@ EXPOSE 80 EXPOSE $arg EXPOSE 8080/tcp 5000`), wantedPorts: nil, - wantedErr: ErrInvalidPort{Match: "EXPOSE $arg"}, + wantedErr: ErrInvalidPort{Match: "$arg"}, }, } @@ -158,6 +159,21 @@ HEALTHCHECK CMD curl -f http://localhost/ || exit 1 Cmd: []string{cmdShell, "curl -f http://localhost/ || exit 1"}, }, }, + "correctly parses multiline healthcheck": { + dockerfile: []byte(` +FROM nginx +HEALTHCHECK --interval=5m\ + --timeout=3s --start-period=2s --retries=3 \ + CMD curl -f http://localhost/ || exit 1 `), + wantedErr: nil, + wantedConfig: &HealthCheck{ + Interval: 300 * time.Second, + Timeout: 3 * time.Second, + StartPeriod: 2 * time.Second, + Retries: 3, + Cmd: []string{cmdShell, "curl -f http://localhost/ || exit 1"}, + }, + }, "correctly parses healthcheck with user's values": { dockerfile: []byte(` FROM nginx @@ -214,15 +230,11 @@ HEALTHCHECK CMD ["a", "b"] }, "healthcheck contains an invalid flag": { dockerfile: []byte(`HEALTHCHECK --interval=5m --randomFlag=4s CMD curl -f http://localhost/ || exit 1`), - wantedErr: fmt.Errorf("parse instructions: Unknown flag: randomFlag"), + wantedErr: fmt.Errorf("parse HEALTHCHECK: flag provided but not defined: -randomFlag"), }, "healthcheck does not contain CMD": { dockerfile: []byte(`HEALTHCHECK --interval=5m curl -f http://localhost/ || exit 1`), - wantedErr: fmt.Errorf("parse instructions: Unknown type \"CURL\" in HEALTHCHECK (try CMD)"), - }, - "healthcheck does not contain command": { - dockerfile: []byte(`HEALTHCHECK --interval=5m CMD`), - wantedErr: fmt.Errorf("parse instructions: Missing command after HEALTHCHECK CMD"), + wantedErr: errors.New("parse HEALTHCHECK: instruction must contain either CMD or NONE"), }, } diff --git a/internal/pkg/docker/dockerfile/errors.go b/internal/pkg/docker/dockerfile/errors.go index b74ff779569..137aad5ad8f 100644 --- a/internal/pkg/docker/dockerfile/errors.go +++ b/internal/pkg/docker/dockerfile/errors.go @@ -11,7 +11,7 @@ type ErrInvalidPort struct { } func (e ErrInvalidPort) Error() string { - return fmt.Sprintf("port represented at %s is invalid or unparseable", e.Match) + return fmt.Sprintf("parse EXPOSE: port represented at %s is invalid or unparseable", e.Match) } // ErrNoExpose means there were no documented EXPOSE statements in the given dockerfile. @@ -20,5 +20,5 @@ type ErrNoExpose struct { } func (e ErrNoExpose) Error() string { - return fmt.Sprintf("no EXPOSE statements in Dockerfile %s", e.Dockerfile) + return fmt.Sprintf("parse EXPOSE: no EXPOSE statements in Dockerfile %s", e.Dockerfile) } diff --git a/internal/pkg/docker/dockerfile/lex.go b/internal/pkg/docker/dockerfile/lex.go new file mode 100644 index 00000000000..7422ad6d104 --- /dev/null +++ b/internal/pkg/docker/dockerfile/lex.go @@ -0,0 +1,242 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dockerfile + +import ( + "bufio" + "fmt" + "io" + "strings" + "unicode" +) + +// instructionName identifies the name of the instruction. +type instructionName int + +const ( + instrErr instructionName = iota // an error occurred while scanning. + instrHealthCheck // a HEALTHCHECK instruction. + instrExpose // an EXPOSE instruction. + instrEOF // done scanning. +) + +const ( + markerExposeInstr = "expose " // start of an EXPOSE instruction. + markerHealthCheckInstr = "healthcheck " // start of a HEALTHCHECK instruction. +) + +var ( + lineContinuationMarkers = []string{"`", "\\"} // denotes that the instruction continues to the next line. + + instrMarkers = map[instructionName]string{ // lookup table for how an instruction starts. + instrExpose: markerExposeInstr, + instrHealthCheck: markerHealthCheckInstr, + } +) + +// An instruction part of a Dockerfile. +// Dockerfiles are of the following format: +// ``` +// # Comment +// INSTRUCTION arguments +// ``` +type instruction struct { + name instructionName // the type of the instruction. + args string // the arguments of an instruction. + line int // line number at the start of this instruction. +} + +// lexer holds the state of the scanner. +type lexer struct { + scanner *bufio.Scanner // line-by-line scanner of the contents of the Dockerfile. + + curLineCount int // line number scanned so far. + curLine string // current line scanned. + curArgs *strings.Builder // accumulated arguments for an instruction. + + instructions chan instruction //channel of discovered instructions. +} + +// lex returns a running lexer that scans the Dockerfile. +// The lexing logic is heavily inspired by: +// https://cs.opensource.google/go/go/+/refs/tags/go1.17.1:src/text/template/parse/lex.go +func lex(reader io.Reader) *lexer { + l := &lexer{ + scanner: bufio.NewScanner(reader), + curArgs: new(strings.Builder), + instructions: make(chan instruction), + } + go l.run() + return l +} + +// next returns the next scanned instruction. +func (lex *lexer) next() instruction { + return <-lex.instructions +} + +// readLine loads the next line in the Dockerfile. +// If we reached the end of the file, then isEOF is set to true. +// If any unexpected error occurs during scanning, then err is not nil. +func (lex *lexer) readLine() (isEOF bool, err error) { + if ok := lex.scanner.Scan(); !ok { + if err := lex.scanner.Err(); err != nil { + return false, err + } + return true, nil + } + lex.curLineCount++ + lex.curLine = lex.scanner.Text() + return false, nil +} + +// emit passes an instruction back to the client. +func (lex *lexer) emit(name instructionName) { + defer lex.curArgs.Reset() + + lex.instructions <- instruction{ + name: name, + args: lex.curArgs.String(), + line: lex.curLineCount, + } +} + +// emitErr notifies clients that an error occurred during scanning. +func (lex *lexer) emitErr(err error) { + lex.instructions <- instruction{ + name: instrErr, + args: err.Error(), + line: lex.curLineCount, + } +} + +// consumeInstr keeps calling readLine and storing the arguments in the lexer until there is no more +// continuation marker and then emits the instruction. +func (lex *lexer) consumeInstr(name instructionName) stateFn { + isEOF, err := lex.readLine() + if err != nil { + lex.emitErr(err) + return nil + } + if isEOF { + lex.emitErr(fmt.Errorf("unexpected EOF while reading Dockerfile at line %d", lex.curLineCount)) + return nil + } + + // For example a healthcheck instruction like: + // ``` + // HEALTHCHECK --interval=5m --timeout=3s --start-period=2s\ + // --retries=3 \ + // CMD curl -f http://localhost/ || exit 1` + // ``` + // will be stored as: + // curArgs = "--interval=5m --timeout=3s --start-period=2s --retries=3 CMD curl -f http://localhost/ || exit 1" + clean := trimContinuationLineMarker(trimLeadingWhitespaces(lex.curLine)) + _, err = lex.curArgs.WriteString(fmt.Sprintf(" %s", clean)) // separate each new line with a space character. + if err != nil { + lex.emitErr(fmt.Errorf("write '%s' to arguments buffer: %w", clean, err)) + return nil + } + + if hasLineContinuationMarker(lex.curLine) { + return lex.consumeInstr(name) + } + lex.emit(name) + return lexContent +} + +// run walks through the state machine for the lexer. +func (lex *lexer) run() { + for state := lexContent; state != nil; { + state = state(lex) + } + close(lex.instructions) +} + +// stateFn represents a state machine transition of the scanner going from one INSTRUCTION to the next. +type stateFn func(*lexer) stateFn + +// lexContent scans until we reach the end of the Dockerfile. +func lexContent(l *lexer) stateFn { + isEOF, err := l.readLine() + if err != nil { + l.emitErr(err) + return nil + } + if isEOF { + l.emit(instrEOF) + return nil + } + line := strings.ToLower(strings.TrimLeftFunc(l.curLine, unicode.IsSpace)) + switch { + case strings.HasPrefix(line, markerExposeInstr): + return lexExpose + case strings.HasPrefix(line, markerHealthCheckInstr): + return lexHealthCheck + default: + return lexContent // Ignore all the other instructions, consume the line without emitting any instructions. + } +} + +// lexExpose collects the arguments for an EXPOSE instruction and then emits it. +func lexExpose(l *lexer) stateFn { + return lexInstruction(l, instrExpose) +} + +// lexHealthCheck collects the arguments for a HEALTHCHECK instruction and then emits it. +func lexHealthCheck(l *lexer) stateFn { + return lexInstruction(l, instrHealthCheck) +} + +// lexInstruction collects all the arguments for the named instruction and then emits it. +func lexInstruction(l *lexer, name instructionName) stateFn { + args := trimContinuationLineMarker(trimInstruction(l.curLine, instrMarkers[name])) + _, err := l.curArgs.WriteString(args) + if err != nil { + l.emitErr(fmt.Errorf("write '%s' to arguments buffer: %w", args, err)) + return nil + } + + if hasLineContinuationMarker(l.curLine) { + return l.consumeInstr(name) + } + l.emit(name) + return lexContent +} + +// hasLineContinuationMarker returns true if the line wraps to the next line. +func hasLineContinuationMarker(line string) bool { + for _, marker := range lineContinuationMarkers { + if strings.HasSuffix(line, marker) { + return true + } + } + return false +} + +// trimInstruction trims the instrMarker prefix from line and returns it. +func trimInstruction(line, instrMarker string) string { + normalized := strings.ToLower(line) + if !strings.Contains(normalized, instrMarker) { + return line + } + idx := strings.Index(normalized, instrMarker) + len(instrMarker) + return line[idx:] +} + +// trimContinuationLineMarker returns the line without any continuation line markers. +// If the line doesn't have a continuation marker, then returns it as is. +func trimContinuationLineMarker(line string) string { + for _, marker := range lineContinuationMarkers { + if strings.HasSuffix(line, marker) { + return strings.TrimSuffix(line, marker) + } + } + return line +} + +// trimLeadingWhitespaces removes any leading space characters. +func trimLeadingWhitespaces(line string) string { + return strings.TrimLeftFunc(line, unicode.IsSpace) +}