pkgdef.go

// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package builder

import (
	"io"
	"path/filepath"
	"regexp"
	"sort"

	"gopkg.in/yaml.v2"

	"go.chromium.org/luci/common/errors"

	"go.chromium.org/luci/cipd/client/cipd/fs"
	"go.chromium.org/luci/cipd/client/cipd/pkg"
	"go.chromium.org/luci/cipd/common"
	"go.chromium.org/luci/cipd/common/cipderr"
)

// PackageDef defines how exactly to build a package.
//
// It specified  what files to put into it, how to name them, how to name
// the package itself, etc. It is loaded from *.yaml file.
type PackageDef struct {
	// Package defines a name of the package.
	Package string

	// Root defines where to search for files. It may either be an absolute path,
	// or it may be a path relative to the package file itself. If omitted, it
	// defaults to "." (i.e., the same directory as the package file)
	Root string

	// InstallMode defines how to deploy the package file: "copy" or "symlink".
	InstallMode pkg.InstallMode `yaml:"install_mode"`

	// PreserveModTime instructs CIPD to preserve the mtime of the files.
	PreserveModTime bool `yaml:"preserve_mtime"`

	// PreserveWritable instructs CIPD to preserve the user-writable permission
	// mode on the files.
	PreserveWritable bool `yaml:"preserve_writable"`

	// Data describes what is deployed with the package.
	Data []PackageChunkDef
}

// PackageChunkDef represents one entry in 'data' section of package definition.
//
// It is either a single file, or a recursively scanned directory (with optional
// list of regexps for files to skip).
type PackageChunkDef struct {
	// Dir is a directory to add to the package (recursively).
	Dir string

	// File is a single file to add to the package.
	File string

	// VersionFile defines where to drop JSON file with package version.
	VersionFile string `yaml:"version_file"`

	// Exclude is a list of regexp patterns to exclude when scanning a directory.
	Exclude []string
}

// LoadPackageDef loads package definition from a YAML source code.
//
// It substitutes %{...} strings in the definition with corresponding values
// from 'vars' map.
func LoadPackageDef(r io.Reader, vars map[string]string) (PackageDef, error) {
	data, err := io.ReadAll(r)
	if err != nil {
		return PackageDef{}, errors.Annotate(err, "reading package definition file").Tag(cipderr.IO).Err()
	}

	out := PackageDef{}
	if err = yaml.Unmarshal(data, &out); err != nil {
		return PackageDef{}, errors.Annotate(err, "bad package definition file").Tag(cipderr.BadArgument).Err()
	}

	// Substitute variables in all strings.
	for _, str := range out.strings() {
		*str, err = subVars(*str, vars)
		if err != nil {
			return PackageDef{}, err
		}
	}

	// Validate global package properties.
	if err = common.ValidatePackageName(out.Package); err != nil {
		return PackageDef{}, err
	}
	if err = pkg.ValidateInstallMode(out.InstallMode); err != nil {
		return PackageDef{}, err
	}

	versionFile := ""
	for i, chunk := range out.Data {
		// Make sure 'dir' and 'file' etc. aren't used together.
		has := make([]string, 0, 3)
		if chunk.File != "" {
			has = append(has, "file")
		}
		if chunk.VersionFile != "" {
			has = append(has, "version_file")
		}
		if chunk.Dir != "" {
			has = append(has, "dir")
		}
		if len(has) == 0 {
			return out, errors.Reason("files entry #%d needs 'file', 'dir' or 'version_file' key", i).Tag(cipderr.BadArgument).Err()
		}
		if len(has) != 1 {
			return out, errors.Reason("files entry #%d should have only one key, got %q", i, has).Tag(cipderr.BadArgument).Err()
		}
		//'version_file' can appear only once, it must be a clean relative path.
		if chunk.VersionFile != "" {
			if versionFile != "" {
				return out, errors.Reason("'version_file' entry can be used only once").Tag(cipderr.BadArgument).Err()
			}
			versionFile = chunk.VersionFile
			if !fs.IsCleanSlashPath(versionFile) {
				return out, errors.Reason("'version_file' must be a path relative to the package root: %s", versionFile).Tag(cipderr.BadArgument).Err()
			}
		}
	}

	// Default 'root' to a directory with the package def file.
	if out.Root == "" {
		out.Root = "."
	}
	return out, nil
}

// FindFiles scans files system and returns files to be added to the package.
//
// It uses a path to package definition file directory ('cwd' argument) to find
// a root of the package.
func (def *PackageDef) FindFiles(cwd string) ([]fs.File, error) {
	// Root of the package is defined relative to package def YAML file.
	absCwd, err := filepath.Abs(cwd)
	if err != nil {
		return nil, errors.Annotate(err, "bad input directory").Tag(cipderr.BadArgument).Err()
	}
	root := filepath.Clean(def.Root)
	if !filepath.IsAbs(root) {
		root = filepath.Join(absCwd, root)
	}

	// Helper to get absolute path to a file given path relative to root.
	makeAbs := func(p string) string {
		return filepath.Join(root, filepath.FromSlash(p))
	}

	// Used to skip duplicates.
	seen := map[string]fs.File{}
	add := func(f fs.File) {
		if seen[f.Name()] == nil {
			seen[f.Name()] = f
		}
	}

	scanOpts := fs.ScanOptions{
		PreserveModTime:  def.PreserveModTime,
		PreserveWritable: def.PreserveWritable,
	}

	for _, chunk := range def.Data {
		// Handled elsewhere.
		if chunk.VersionFile != "" {
			continue
		}

		// Individual file.
		if chunk.File != "" {
			file, err := fs.WrapFile(makeAbs(chunk.File), root, nil, scanOpts)
			if err != nil {
				return nil, err
			}
			add(file)
			continue
		}

		// A subdirectory to scan (with filtering).
		if chunk.Dir != "" {
			// Absolute path to directory to scan.
			startDir := makeAbs(chunk.Dir)
			// Exclude files as specified in 'exclude' section.
			exclude, err := makeExclusionFilter(chunk.Exclude)
			if err != nil {
				return nil, errors.Annotate(err, "dir %q", chunk.Dir).Err()
			}
			// Run the scan.
			files, err := fs.ScanFileSystem(startDir, root, exclude, scanOpts)
			if err != nil {
				return nil, errors.Annotate(err, "dir %q", chunk.Dir).Err()
			}
			for _, f := range files {
				add(f)
			}
			continue
		}

		// LoadPackageDef does validation, so this should not happen.
		return nil, errors.Reason("unexpected definition: %v", chunk).Tag(cipderr.BadArgument).Err()
	}

	// Sort by Name().
	names := make([]string, 0, len(seen))
	for n := range seen {
		names = append(names, n)
	}
	sort.Strings(names)

	// Final sorted array of fs.File.
	out := make([]fs.File, 0, len(names))
	for _, n := range names {
		out = append(out, seen[n])
	}
	return out, nil
}

// VersionFile defines where to drop JSON file with package version.
func (def *PackageDef) VersionFile() string {
	// It is already validated by LoadPackageDef, so just return it.
	for _, chunk := range def.Data {
		if chunk.VersionFile != "" {
			return chunk.VersionFile
		}
	}
	return ""
}

// makeExclusionFilter produces a predicate that checks a relative file path
// against a list of regexps and returns true to exclude it.
//
// Note that regexps are defined against slash-separated relative paths (to make
// the package definition YAML platform-agnostic).
func makeExclusionFilter(patterns []string) (fs.ScanFilter, error) {
	if len(patterns) == 0 {
		return nil, nil
	}

	// Compile regular expressions. Note that we want to verify that each
	// individual pattern is a valid regexp. For that reason we don't just
	// concatenate them in a single uber-regexp and compile it afterwards.
	exps := []*regexp.Regexp{}
	for _, expr := range patterns {
		if expr == "" {
			continue
		}
		if expr[0] != '^' {
			expr = "^" + expr
		}
		if expr[len(expr)-1] != '$' {
			expr = expr + "$"
		}
		re, err := regexp.Compile(expr)
		if err != nil {
			return nil, errors.Annotate(err, "bad exclusion pattern").Tag(cipderr.BadArgument).Err()
		}
		exps = append(exps, re)
	}

	return func(rel string) bool {
		rel = filepath.ToSlash(rel)
		for _, exp := range exps {
			if exp.MatchString(rel) {
				return true
			}
		}
		return false
	}, nil
}

////////////////////////////////////////////////////////////////////////////////
// Variable substitution.

var subVarsRe = regexp.MustCompile(`\$\{[^\}]+\}`)

// strings return array of pointers to all strings in PackageDef that can
// contain ${var} variables.
func (def *PackageDef) strings() []*string {
	out := []*string{
		&def.Package,
		&def.Root,
	}
	// Important to use index here, to get a point to a real object, not its copy.
	for i := range def.Data {
		out = append(out, def.Data[i].strings()...)
	}
	return out
}

// strings return array of pointers to all strings in PackageChunkDef that can
// contain ${var} variables.
func (def *PackageChunkDef) strings() []*string {
	out := []*string{
		&def.Dir,
		&def.File,
		&def.VersionFile,
	}
	for i := range def.Exclude {
		out = append(out, &def.Exclude[i])
	}
	return out
}

// subVars replaces "${key}" in strings with values from 'vars' map. Returns
// error if some keys weren't found in 'vars' map.
func subVars(s string, vars map[string]string) (string, error) {
	var badKeys []string
	res := subVarsRe.ReplaceAllStringFunc(s, func(match string) string {
		// Strip '${' and '}'.
		key := match[2 : len(match)-1]
		val, ok := vars[key]
		if !ok {
			badKeys = append(badKeys, key)
			return match
		}
		return val
	})
	if len(badKeys) != 0 {
		return res, errors.Reason("values for some variables are not provided: %v", badKeys).Tag(cipderr.BadArgument).Err()
	}
	return res, nil
}