Localhost pathing

Signed-off-by: John Reese <john@reese.dev>

.circleci/config.yml

@@ -34,6 +34,7 @@ jobs:
 
     steps:
       - checkout
+      - setup_remote_docker
       - restore_cache:
           keys:
             - go-mod-{{ checksum "go.sum" }}

acceptance.bats

@@ -324,6 +324,7 @@
   [ "$status" -eq 0 ]
 }
 
-@test "Can push to registry" {
-  run docker run -p 5000:5000 registry --name conftest-registry-test
+@test "Can push and pull bundles from registry" {
+  run ./scripts/push-pull-e2e.sh
+  [ "$status" -eq 0 ]
 }

downloader/downloader.go

@@ -3,6 +3,7 @@ package downloader
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	getter "github.com/hashicorp/go-getter"
 )
@@ -32,9 +33,14 @@ var getters = map[string]getter.Getter{
 func Download(ctx context.Context, dst string, urls []string) error {
 	opts := []getter.ClientOption{}
 	for _, url := range urls {
+		detectedURL, err := Detect(url, dst)
+		if err != nil {
+			return fmt.Errorf("detecting url: %w", err)
+		}
+
 		client := &getter.Client{
 			Ctx:       ctx,
-			Src:       url,
+			Src:       detectedURL,
 			Dst:       dst,
 			Pwd:       dst,
 			Mode:      getter.ClientModeAny,
@@ -54,6 +60,16 @@ func Download(ctx context.Context, dst string, urls []string) error {
 // Detect determines whether a url is a known source url from which we can download files.
 // If a known source is found, the url is formatted, otherwise an error is returned.
 func Detect(url string, dst string) (string, error) {
+
+	// localhost is not considered a valid scheme for the detector which
+	// causes pull commands that reference localhost to error.
+	//
+	// To allow for localhost to be used, replace the localhost reference
+	// with the IP address.
+	if strings.Contains(url, "localhost") {
+		url = strings.ReplaceAll(url, "localhost", "127.0.0.1")
+	}
+
 	result, err := getter.Detect(url, dst, detectors)
 	if err != nil {
 		return "", fmt.Errorf("detect: %w", err)

downloader/detect_oci.go → downloader/oci_detector.go

@@ -6,13 +6,6 @@ import (
 	"strings"
 )
 
-var matchRegistries = []*regexp.Regexp{
-	regexp.MustCompile("azurecr.io"),
-	regexp.MustCompile("gcr.io"),
-	regexp.MustCompile("registry.gitlab.com"),
-	regexp.MustCompile("[0-9]{12}.dkr.ecr.[a-z0-9-]*.amazonaws.com"),
-}
-
 // OCIDetector implements Detector to detect OCI registry URLs and turn
 // them into URLs that the OCI getter can understand.
 type OCIDetector struct{}
@@ -24,7 +17,7 @@ func (d *OCIDetector) Detect(src, _ string) (string, bool, error) {
 	}
 
 	if containsOCIRegistry(src) || containsLocalRegistry(src) {
-		url, err := d.detectHTTP(src)
+		url, err := detectHTTP(src)
 		if err != nil {
 			return "", false, fmt.Errorf("detect http: %w", err)
 		}
@@ -36,6 +29,13 @@ func (d *OCIDetector) Detect(src, _ string) (string, bool, error) {
 }
 
 func containsOCIRegistry(src string) bool {
+	var matchRegistries = []*regexp.Regexp{
+		regexp.MustCompile("azurecr.io"),
+		regexp.MustCompile("gcr.io"),
+		regexp.MustCompile("registry.gitlab.com"),
+		regexp.MustCompile("[0-9]{12}.dkr.ecr.[a-z0-9-]*.amazonaws.com"),
+	}
+
 	for _, matchRegistry := range matchRegistries {
 		if matchRegistry.MatchString(src) {
 			return true
@@ -49,28 +49,23 @@ func containsLocalRegistry(src string) bool {
 	return strings.Contains(src, "127.0.0.1:5000") || strings.Contains(src, "localhost:5000")
 }
 
-func (d *OCIDetector) detectHTTP(src string) (string, error) {
+func detectHTTP(src string) (string, error) {
 	parts := strings.Split(src, "/")
 	if len(parts) < 2 {
-		return "", fmt.Errorf("URL is not a valid Azure registry URL")
+		return "", fmt.Errorf("URL is not a valid registry URL")
 	}
 
-	return "oci://" + getRepositoryFromURL(src), nil
+	repository := getRepositoryFromURL(src)
+	return "oci://" + repository, nil
 }
 
 func getRepositoryFromURL(url string) string {
-	if repositoryContainsTag(url) {
+	pathParts := strings.Split(url, "/")
+	lastPathPart := pathParts[len(pathParts)-1]
+
+	if strings.Contains(lastPathPart, ":") {
 		return url
 	}
 
 	return url + ":latest"
 }
-
-func repositoryContainsTag(repository string) bool {
-	path := strings.Split(repository, "/")
-	return pathContainsTag(path[len(path)-1])
-}
-
-func pathContainsTag(path string) bool {
-	return strings.Contains(path, ":")
-}

downloader/get_oci.go → downloader/oci_getter.go

@@ -27,12 +27,7 @@ func (g *OCIGetter) ClientMode(u *url.URL) (getter.ClientMode, error) {
 func (g *OCIGetter) Get(path string, u *url.URL) error {
 	ctx := g.Context()
 
-	if !pathContainsTag(u.Path) {
-		u.Path = u.Path + ":latest"
-	}
-
-	err := os.MkdirAll(path, os.ModePerm)
-	if err != nil {
+	if err := os.MkdirAll(path, os.ModePerm); err != nil {
 		return fmt.Errorf("make policy directory: %w", err)
 	}
 
@@ -49,8 +44,10 @@ func (g *OCIGetter) Get(path string, u *url.URL) error {
 	fileStore := content.NewFileStore(path)
 	defer fileStore.Close()
 
-	repository := u.Host + u.Path
-	_, _, err = oras.Pull(ctx, resolver, repository, fileStore)
+	repository := getRepositoryFromURL(u.Path)
+	pullURL := u.Host + repository
+
+	_, _, err = oras.Pull(ctx, resolver, pullURL, fileStore)
 	if err != nil {
 		return fmt.Errorf("pulling policy: %w", err)
 	}

internal/commands/push.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -30,7 +31,7 @@ Optionally, a tag can be specified, e.g.:
 
 Optionally, specific directory can be passed as a second argument, e.g.:
 
-	$ conftest push instrumenta.azurecr.io/my-registry:v1 /path/to/dir
+	$ conftest push instrumenta.azurecr.io/my-registry:v1 path/to/dir
 
 Conftest leverages the ORAS library under the hood. This allows arbitrary artifacts to 
 be stored in compatible OCI registries. Currently open policy agent bundles are supported by 
@@ -39,7 +40,7 @@ the docker/distribution (https://github.com/docker/distribution) registry and by
 The policy location defaults to the policy directory in the local folder.
 The location can be overridden with the '--policy' flag, e.g.:
 
-	$ conftest push --policy <my-directory> <oci-url>
+	$ conftest push --policy <my-directory> url
 `
 
 const (
@@ -52,10 +53,10 @@ const (
 // bundles to an OCI registry.
 func NewPushCommand(ctx context.Context, logger *log.Logger) *cobra.Command {
 	cmd := cobra.Command{
-		Use:   "push <repository> [filepath]",
-		Short: "Upload OPA bundles to an OCI registry",
+		Use:   "push <repository>",
+		Short: "Push OPA bundles to an OCI registry",
 		Long:  pushDesc,
-		Args:  cobra.RangeArgs(1, 2),
+		Args:  cobra.ExactArgs(1),
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			if err := viper.BindPFlag("policy", cmd.Flags().Lookup("policy")); err != nil {
 				return fmt.Errorf("bind flag: %w", err)
@@ -65,7 +66,20 @@ func NewPushCommand(ctx context.Context, logger *log.Logger) *cobra.Command {
 		},
 
 		RunE: func(cmd *cobra.Command, args []string) error {
+
 			repository := args[0]
+			if !strings.Contains(repository, "/") {
+				return errors.New("destination url missing repository")
+			}
+
+			// When the destination repository to push to does not contain a
+			// tag, append the latest tag so the bundle is not pushed without
+			// a tag.
+			pathParts := strings.Split(repository, "/")
+			lastPathPart := pathParts[len(pathParts)-1]
+			if !strings.Contains(lastPathPart, ":") {
+				repository = repository + ":latest"
+			}
 
 			logger.Printf("pushing bundle to: %s", repository)
 			manifest, err := pushBundle(ctx, repository, viper.GetString("policy"))
@@ -100,15 +114,8 @@ func pushBundle(ctx context.Context, repository string, path string) (*ocispec.D
 		return nil, fmt.Errorf("building layers: %w", err)
 	}
 
-	var repositoryWithTag string
-	if strings.Contains(repository, ":") {
-		repositoryWithTag = repository
-	} else {
-		repositoryWithTag = repository + ":latest"
-	}
-
 	extraOpts := []oras.PushOpt{oras.WithConfigMediaType(openPolicyAgentConfigMediaType)}
-	manifest, err := oras.Push(ctx, resolver, repositoryWithTag, memoryStore, layers, extraOpts...)
+	manifest, err := oras.Push(ctx, resolver, repository, memoryStore, layers, extraOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("pushing manifest: %w", err)
 	}
@@ -117,11 +124,7 @@ func pushBundle(ctx context.Context, repository string, path string) (*ocispec.D
 }
 
 func buildLayers(ctx context.Context, memoryStore *content.Memorystore, path string) ([]ocispec.Descriptor, error) {
-	loader := policy.Loader{
-		PolicyPaths: []string{path},
-		DataPaths:   []string{path},
-	}
-	engine, err := loader.Load(ctx)
+	engine, err := policy.LoadWithData(ctx, []string{path}, []string{path})
 	if err != nil {
 		return nil, fmt.Errorf("load: %w", err)
 	}

internal/runner/test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	"github.com/open-policy-agent/conftest/downloader"
 	"github.com/open-policy-agent/conftest/output"
 	"github.com/open-policy-agent/conftest/parser"
 	"github.com/open-policy-agent/conftest/policy"
@@ -45,12 +46,15 @@ func (t *TestRunner) Run(ctx context.Context, fileList []string) ([]output.Check
 		return nil, fmt.Errorf("get configurations: %w", err)
 	}
 
-	loader := policy.Loader{
-		DataPaths:   t.Data,
-		PolicyPaths: t.Policy,
-		URLs:        t.Update,
+	// When there are policies to download, they are currently placed in the first
+	// directory that appears in the list of policies.
+	if len(t.Update) > 0 {
+		if err := downloader.Download(ctx, t.Policy[0], t.Update); err != nil {
+			return nil, fmt.Errorf("update policies: %w", err)
+		}
 	}
-	engine, err := loader.Load(ctx)
+
+	engine, err := policy.LoadWithData(ctx, t.Policy, t.Data)
 	if err != nil {
 		return nil, fmt.Errorf("load: %w", err)
 	}

internal/runner/verify.go

@@ -20,13 +20,9 @@ type VerifyRunner struct {
 	Trace   bool
 }
 
-// Run executes the Rego tests at the given PolicyPath(s)
+// Run executes the Rego tests for the given policies.
 func (r *VerifyRunner) Run(ctx context.Context) ([]output.CheckResult, error) {
-	loader := policy.Loader{
-		DataPaths:   r.Data,
-		PolicyPaths: r.Policy,
-	}
-	engine, err := loader.Load(ctx)
+	engine, err := policy.LoadWithData(ctx, r.Policy, r.Data)
 	if err != nil {
 		return nil, fmt.Errorf("load: %w", err)
 	}

policy/engine.go

@@ -4,14 +4,17 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
 	"github.com/open-policy-agent/conftest/output"
 	"github.com/open-policy-agent/conftest/parser"
 
 	"github.com/open-policy-agent/opa/ast"
+	"github.com/open-policy-agent/opa/loader"
 	"github.com/open-policy-agent/opa/rego"
 	"github.com/open-policy-agent/opa/storage"
 	"github.com/open-policy-agent/opa/topdown"
@@ -27,6 +30,83 @@ type Engine struct {
 	docs     map[string]string
 }
 
+// Load returns an Engine after loading all of the specified policies.
+func Load(ctx context.Context, policyPaths []string) (*Engine, error) {
+	policies, err := loader.AllRegos(policyPaths)
+	if err != nil {
+		return nil, fmt.Errorf("load: %w", err)
+	} else if len(policies.Modules) == 0 {
+		return nil, fmt.Errorf("no policies found in %v", policyPaths)
+	}
+
+	compiler, err := policies.Compiler()
+	if err != nil {
+		return nil, fmt.Errorf("get compiler: %w", err)
+	}
+
+	policyContents := make(map[string]string)
+	for path, module := range policies.ParsedModules() {
+		path = filepath.Clean(path)
+		path = filepath.ToSlash(path)
+
+		policyContents[path] = module.String()
+	}
+
+	engine := Engine{
+		modules:  policies.ParsedModules(),
+		compiler: compiler,
+		policies: policyContents,
+	}
+
+	return &engine, nil
+}
+
+// Load returns an Engine after loading all of the specified policies and data paths.
+func LoadWithData(ctx context.Context, policyPaths []string, dataPaths []string) (*Engine, error) {
+	engine, err := Load(ctx, policyPaths)
+	if err != nil {
+		return nil, fmt.Errorf("loading policies: %w", err)
+	}
+
+	// FilteredPaths will recursively find all file paths that contain a valid document
+	// extension from the given list of data paths.
+	allDocumentPaths, err := loader.FilteredPaths(dataPaths, func(abspath string, info os.FileInfo, depth int) bool {
+		if info.IsDir() {
+			return false
+		}
+		return !contains([]string{".yaml", ".yml", ".json"}, filepath.Ext(info.Name()))
+	})
+	if err != nil {
+		return nil, fmt.Errorf("filter data paths: %w", err)
+	}
+
+	documents, err := loader.NewFileLoader().All(allDocumentPaths)
+	if err != nil {
+		return nil, fmt.Errorf("load documents: %w", err)
+	}
+	store, err := documents.Store()
+	if err != nil {
+		return nil, fmt.Errorf("get documents store: %w", err)
+	}
+
+	documentContents := make(map[string]string)
+	for _, documentPath := range allDocumentPaths {
+		contents, err := ioutil.ReadFile(documentPath)
+		if err != nil {
+			return nil, fmt.Errorf("read file: %w", err)
+		}
+
+		documentPath = filepath.Clean(documentPath)
+		documentPath = filepath.ToSlash(documentPath)
+		documentContents[documentPath] = string(contents)
+	}
+
+	engine.store = store
+	engine.docs = documentContents
+
+	return engine, nil
+}
+
 // Check executes all of the loaded policies against the input and returns the results.
 func (e *Engine) Check(ctx context.Context, configs map[string]interface{}, namespace string) ([]output.CheckResult, error) {
 	var checkResults []output.CheckResult