awscli_test.go

package gofakes3_test

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os/exec"
	"path"
	"reflect"
	"regexp"
	"strings"
	"testing"
	"time"

	"github.com/rclone/gofakes3"
)

func TestCLILsBuckets(t *testing.T) {
	cli := newTestCLI(t, withoutInitialBuckets())
	defer cli.Close()

	if len(cli.lsBuckets()) != 0 {
		t.Fatal()
	}

	cli.backendCreateBucket("foo")
	if !reflect.DeepEqual(cli.lsBuckets().Names(), []string{"foo"}) {
		t.Fatal()
	}

	cli.backendCreateBucket("bar")
	if !reflect.DeepEqual(cli.lsBuckets().Names(), []string{"bar", "foo"}) {
		t.Fatal()
	}
}

func TestCLILsFiles(t *testing.T) {
	cli := newTestCLI(t)
	defer cli.Close()

	if len(cli.lsFiles(defaultBucket, "")) != 0 {
		t.Fatal()
	}

	cli.backendPutString(defaultBucket, "test-one", nil, "hello")
	cli.assertLsFiles(defaultBucket, "",
		nil, []string{"test-one"})

	cli.backendPutString(defaultBucket, "test-two", nil, "hello")
	cli.assertLsFiles(defaultBucket, "",
		nil, []string{"test-one", "test-two"})

	// only "test-one" and "test-two" should pass the prefix match
	cli.backendPutString(defaultBucket, "no-match", nil, "hello")
	cli.assertLsFiles(defaultBucket, "test-",
		nil, []string{"test-one", "test-two"})

	cli.backendPutString(defaultBucket, "test/yep", nil, "hello")
	cli.assertLsFiles(defaultBucket, "",
		[]string{"test/"}, []string{"no-match", "test-one", "test-two"})

	// "test-one" and "test-two" and the directory "test" should pass the prefix match:
	cli.assertLsFiles(defaultBucket, "test",
		[]string{"test/"}, []string{"test-one", "test-two"})

	// listing with a trailing slash should list the directory contents:
	cli.assertLsFiles(defaultBucket, "test/",
		nil, []string{"yep"})
}

func TestCLIRmOne(t *testing.T) {
	cli := newTestCLI(t)
	defer cli.Close()

	cli.backendPutString(defaultBucket, "foo", nil, "hello")
	cli.backendPutString(defaultBucket, "bar", nil, "hello")
	cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar"})

	cli.rm(cli.fileArg(defaultBucket, "foo"))
	cli.assertLsFiles(defaultBucket, "", nil, []string{"bar"})
}

func TestCLIRmMulti(t *testing.T) {
	cli := newTestCLI(t)
	defer cli.Close()

	cli.backendPutString(defaultBucket, "foo", nil, "hello")
	cli.backendPutString(defaultBucket, "bar", nil, "hello")
	cli.backendPutString(defaultBucket, "baz", nil, "hello")
	cli.assertLsFiles(defaultBucket, "", nil, []string{"foo", "bar", "baz"})

	cli.rmMulti(defaultBucket, "foo", "bar", "baz")
	cli.assertLsFiles(defaultBucket, "", nil, nil)
}

func TestCLIDownload(t *testing.T) {
	// NOTE: this must be set to the largest value you plan to test in the test cases.
	var source = randomFileBody(100000000)

	for _, tc := range []struct {
		in []byte
	}{
		{in: nil},
		{in: source[:1]},

		// FIXME: Beyond a certain size, the AWS client switches to using range
		// requests and downloads several parts in parallel. This takes a stab
		// at what that size is, but it isn't an especially robust way to
		// determine what the spill point is:
		{in: source[:1000000]},
		{in: source[:10000000]},
		{in: source[:100000000]},
	} {
		t.Run("", func(t *testing.T) {
			cli := newTestCLI(t)
			defer cli.Close()

			cli.backendPutBytes(defaultBucket, "foo", nil, tc.in)
			out := cli.download(defaultBucket, "foo")
			if !bytes.Equal(out, tc.in) {
				t.Fatal()
			}
		})
	}
}

type testCLI struct {
	*testServer
}

func newTestCLI(t *testing.T, options ...testServerOption) *testCLI {
	return &testCLI{newTestServer(t, options...)}
}

func (tc *testCLI) command(method string, subcommand string, args ...string) *exec.Cmd {
	tc.Helper()

	if method != "s3" && method != "s3api" {
		panic("expected 's3' or 's3api'")
	}

	cmdArgs := append([]string{
		"--output", "json",
		method,
		"--endpoint", tc.server.URL,
		subcommand,
	}, args...)

	cmd := exec.Command("aws", cmdArgs...)

	log.Println("cli args:", cmdArgs)

	cmd.Env = []string{
		"AWS_ACCESS_KEY_ID=key",
		"AWS_SECRET_ACCESS_KEY=secret",
		"AWS_EC2_METADATA_DISABLED=true", // See: https://github.com/aws/aws-cli/issues/5234#issuecomment-705831465
	}
	return cmd
}

func (tc *testCLI) run(method string, subcommand string, args ...string) {
	tc.Helper()
	err := tc.command(method, subcommand, args...).Run()
	if _, ok := err.(*exec.Error); ok {
		tc.Skip("aws cli not found on $PATH")
	}
	tc.OK(err)
}

func (tc *testCLI) combinedOutput(method string, subcommand string, args ...string) (out []byte) {
	tc.Helper()
	out, err := tc.command(method, subcommand, args...).CombinedOutput()
	if _, ok := err.(*exec.Error); ok {
		tc.Skip("aws cli not found on $PATH")
	}
	if err != nil {
		tc.Log(string(out))
	}
	tc.OK(err)
	return out
}

var cliLsDirMatcher = regexp.MustCompile(`^\s*PRE (.*)$`)

func (tc *testCLI) assertLsFiles(bucket string, prefix string, dirs []string, files []string) (items lsItems) {
	tc.Helper()
	items = tc.lsFiles(bucket, prefix)
	items.assertContents(tc.TT, dirs, files)
	return items
}

func (tc *testCLI) lsFiles(bucket string, prefix string) (items lsItems) {
	tc.Helper()

	prefix = strings.TrimLeft(prefix, "/")
	out := tc.combinedOutput("s3", "ls", fmt.Sprintf("s3://%s/%s", bucket, prefix))

	scn := bufio.NewScanner(bytes.NewReader(out))
	for scn.Scan() {
		cur := scn.Text()
		dir := cliLsDirMatcher.FindStringSubmatch(cur)
		if dir != nil {
			items = append(items, lsItem{
				isDir: true,
				name:  dir[1], // first submatch
			})

		} else { // file matching
			var ct cliTime
			var item lsItem
			tc.OKAll(fmt.Sscan(scn.Text(), &ct, &item.size, &item.name))
			item.date = time.Time(ct)
			items = append(items, item)
		}
	}

	return items
}

func (tc *testCLI) lsBuckets() (buckets gofakes3.Buckets) {
	tc.Helper()

	out := tc.combinedOutput("s3", "ls")

	scn := bufio.NewScanner(bytes.NewReader(out))
	for scn.Scan() {
		var ct cliTime
		var b gofakes3.BucketInfo
		tc.OKAll(fmt.Sscan(scn.Text(), &ct, &b.Name))
		b.CreationDate = ct.contentTime()
		buckets = append(buckets, b)
	}

	return buckets
}

func (tc *testCLI) download(bucket, object string) []byte {
	tc.Helper()
	return tc.combinedOutput("s3", "cp", fmt.Sprintf("s3://%s/%s", bucket, object), "-")
}

func (tc *testCLI) rmMulti(bucket string, objects ...string) {
	tc.Helper()

	// delete-objects --bucket fakes3 --delete 'Objects=[{Key=test},{Key=test2}]'

	var delArg struct{ Objects []gofakes3.ObjectID }
	for _, obj := range objects {
		delArg.Objects = append(delArg.Objects, gofakes3.ObjectID{Key: obj})
	}
	bts, err := json.Marshal(delArg)
	if err != nil {
		panic(err)
	}

	args := []string{
		"--bucket", bucket,
		"--delete", string(bts),
	}
	tc.run("s3api", "delete-objects", args...)
}

func (tc *testCLI) rm(fileURL string) {
	tc.Helper()
	tc.run("s3", "rm", fileURL)
}

func (tc *testCLI) fileArg(bucket string, file string) string {
	return fmt.Sprintf("s3://%s", path.Join(bucket, file))
}

type cliTime time.Time

func (c cliTime) contentTime() gofakes3.ContentTime {
	return gofakes3.NewContentTime(time.Time(c))
}

func (c *cliTime) Scan(state fmt.ScanState, verb rune) error {
	d, err := state.Token(false, nil)
	if err != nil {
		return err
	}
	ds := string(d)

	t, err := state.Token(true, nil)
	if err != nil {
		return err
	}
	ts := string(t)

	// CLI returns time in the machine's timezone:
	tv, err := time.ParseInLocation("2006-01-01 15:04:05", ds+" "+ts, time.Local)
	if err != nil {
		return err
	}

	*c = cliTime(tv.In(time.UTC))

	return nil
}