feature: cli s3 integration

.travis.yml

@@ -11,10 +11,6 @@ env:
 
 matrix:
   include:  # WARNING remove -cpu=* with TEST_WITH_REAL_API=1
-    - go:   1.5.3
-      env:  GO15VENDOREXPERIMENT=1 TEST_WITH_REAL_API=0 GOTESTFLAGS="-race -cpu=1,2,4"
-    - go:   1.6
-      env:  GO15VENDOREXPERIMENT=1 TEST_WITH_REAL_API=0 GOTESTFLAGS="-race -cpu=1,2,4"
     - go:   1.7
       env:  TEST_WITH_REAL_API=0 GOTESTFLAGS="-race -cpu=1,2,4"
     - go:   1.8

Makefile

@@ -91,8 +91,8 @@ prepare-release-dist: build
 	GOOS=linux  GOARCH=arm    go build -o dist/latest/scw-linux-arm         github.com/scaleway/scaleway-cli/cmd/scw
 	GOOS=linux  GOARCH=arm64  go build -o dist/latest/scw-linux-arm64       github.com/scaleway/scaleway-cli/cmd/scw
 
-	GOOS=darwin  GOARCH=386   go build -o dist/latest/scw-darwin-i386       github.com/scaleway/scaleway-cli/cmd/scw
-	GOOS=darwin  GOARCH=amd64 go build -o dist/latest/scw-darwin-amd64      github.com/scaleway/scaleway-cli/cmd/scw
+	GOOS=darwin  GOARCH=386   go build --tags kqueue -o dist/latest/scw-darwin-i386       github.com/scaleway/scaleway-cli/cmd/scw
+	GOOS=darwin  GOARCH=amd64 go build --tags kqueue -o dist/latest/scw-darwin-amd64      github.com/scaleway/scaleway-cli/cmd/scw
 
 	GOOS=freebsd GOARCH=386   go build -o dist/latest/scw-freebsd-i386      github.com/scaleway/scaleway-cli/cmd/scw
 	GOOS=freebsd GOARCH=amd64 go build -o dist/latest/scw-freebsd-amd64     github.com/scaleway/scaleway-cli/cmd/scw

README.md

@@ -52,6 +52,7 @@ Read the [blog post](https://blog.scaleway.com/2015/05/20/manage-baremetal-serve
     * [`rm [OPTIONS] SERVER [SERVER...]`](#scw-rm)
     * [`rmi [OPTIONS] IMAGE [IMAGE...]`](#scw-rmi)
     * [`run [OPTIONS] IMAGE [COMMAND] [ARGS...]`](#scw-run)
+    * [`s3 [OPTIONS]`](#scw-s3)
     * [`search [OPTIONS] TERM`](#scw-search)
     * [`start [OPTIONS] SERVER [SERVER...]`](#scw-start)
     * [`stop [OPTIONS] SERVER [SERVER...]`](#scw-stop)
@@ -207,6 +208,7 @@ Commands:
     rm        Remove one or more servers
     rmi       Remove one or more image(s)/volume(s)/snapshot(s)
     run       Run a command in a new server
+    s3        Access to s3 bucket
     search    Search the Scaleway Hub for images
     start     Start a stopped server
     stop      Stop a running server
@@ -780,6 +782,80 @@ Examples:
  ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
 ```
 
+#### `scw s3`
+
+Integration of object storage to the scaleway CLI is done using the minio client. This is not a long term solution and we think about providing our own S3 cli, but it will probably not be compliant with minio client.
+Please understand that we will put priority on innovation regarding our cloud rather than doing another full s3 client.
+
+ ```console
+ NAME:
+   scw s3 - Minio Client for cloud storage and filesystems.
+
+ USAGE:
+   scw s3 [FLAGS] COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...]
+
+ COMMANDS:
+   ls       List files and folders.
+   mb       Make a bucket or a folder.
+   cat      Display file and object contents.
+   pipe     Redirect STDIN to an object or file or STDOUT.
+   share    Generate URL for sharing.
+   cp       Copy files and objects.
+   mirror   Mirror buckets and folders.
+   find     Search for files and objects.
+   stat     Stat contents of objects and folders.
+   diff     List differences in object name, size, and date between folders.
+   rm       Remove files and objects.
+   session  Manage saved sessions for cp command.
+   config   Manage mc configuration file.
+
+ GLOBAL FLAGS:
+   --config-folder value, -C value  Path to configuration folder. (default: "/home/atom/.mc")
+   --quiet, -q                      Disable progress bar display.
+   --no-color                       Disable color theme.
+   --json                           Enable JSON formatted output.
+   --debug                          Enable debug output.
+   --insecure                       Disable SSL certificate verification.
+   --help, -h                       Show help.
+```
+
+To begin with, you will need to configure your credentials, using the following command:
+
+```console
+$> scw s3 config
+AccessKey: <YOUR-ACCESS-KEY>
+SecretKey: <YOUR-SECRET-KEY>
+```
+
+This call will configure two minio profiles, ams and par, respectively for 'nl-ams' and
+'fr-par' regions, currently, the 'fr-par' region is not usable, but will be soon.
+
+The following examples show basic s3 operations on 'nl-ams' region:
+
+Creating a bucket:
+```console
+$> scw s3 mb ams/minio-test-bucket
+Bucket created successfully `ams/minio-test-bucket`.
+```
+
+Listing:
+```console
+$> scw s3 ls ams/
+[2018-11-20 11:06:32 CET]     0B minio-test-bucket/
+```
+
+Put an object:
+```console
+$> scw s3 cp /etc/hosts ams/minio-test-bucket/hosts
+`/etc/hosts` -> `ams/minio-test-bucket/hosts`
+```
+
+Removing an object / bucket:
+```console
+$> scw s3 rm ams/minio-test-bucket/hosts           
+Removing `ams/minio-test-bucket/hosts`.
+```
+
 #### `scw search`
 
 ```console
@@ -1245,7 +1321,7 @@ View full [commits list](https://github.com/scaleway/scaleway-cli/compare/v1.17.
 
 * Fix: it is now possible to `scw inspect` snapshots [#510](https://github.com/scaleway/scaleway-cli/issues/510)
 * Fix: add region details in cache for snapshots, images, bootscripts and volumes [#510](https://github.com/scaleway/scaleway-cli/issues/510)
-* Fix: `scw attach` works again [#508](https://github.com/scaleway/scaleway-cli/issues/508) 
+* Fix: `scw attach` works again [#508](https://github.com/scaleway/scaleway-cli/issues/508)
 * Fix unittests
 
 View full [commits list](https://github.com/scaleway/scaleway-cli/compare/v1.16...v1.17)
@@ -1655,6 +1731,7 @@ For previous Node.js versions, see [scaleway-cli-node](https://github.com/moul/s
   * Support of `run -e, --env` option
   * Support of `run --name` option
   * Support of `run -v, --volume` option
+* Support of `s3` command
 * Support of `search` command
   * Support of `search --no-trunc` option
 * Support of `start` command

pkg/cli/cmd_s3.go

@@ -0,0 +1,39 @@
+// Copyright (C) 2015 Scaleway. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE.md file.
+
+package cli
+
+import (
+	"github.com/scaleway/scaleway-cli/pkg/commands"
+)
+
+var cmdS3 = &Command{
+	Exec:        runS3,
+	UsageLine:   "s3 [OPTIONS]",
+	Description: "Access to s3 buckets",
+	Help:        "Access to s3 buckets.",
+}
+
+func init() {
+	cmdS3.Flag.BoolVar(&s3Help, []string{"h", "-help"}, false, "Print usage")
+}
+
+// Flags
+var s3Help bool // -h, --help flag
+
+func runS3(cmd *Command, rawArgs []string) error {
+	var args commands.S3Args
+	if s3Help {
+		args = commands.S3Args{
+			Command: make([]string, 0),
+		}
+		args.Command = append(args.Command, "--help")
+	} else {
+		args = commands.S3Args{
+			Command: rawArgs,
+		}
+	}
+	ctx := cmd.GetContext(rawArgs)
+	return commands.S3(ctx, args)
+}

pkg/cli/commands.go

@@ -33,6 +33,7 @@ var Commands = []*Command{
 	cmdRm,
 	cmdRmi,
 	cmdRun,
+	cmdS3,
 	cmdSearch,
 	cmdStart,
 	cmdStop,

pkg/commands/s3.go

@@ -0,0 +1,154 @@
+// Copyright (C) 2015 Scaleway. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE.md file.
+
+package commands
+
+import (
+	"encoding/json"
+	"fmt"
+	minio "github.com/minio/mc/cmd"
+	"github.com/scaleway/scaleway-cli/pkg/config"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type clientHost struct {
+	Url       string `json:"url"`
+	AccessKey string `json:"accessKey"`
+	SecretKey string `json:"secretKey"`
+	Api       string `json:"api"`
+	Lookup    string `json:"lookup"`
+}
+
+type clientConfig struct {
+	Version string `json:"version"`
+
+	Hosts map[string]clientHost `json:"hosts"`
+}
+
+// S3Args are flags for the `runS3` function
+type S3Args struct {
+	Command []string // command line for minio client
+}
+
+// getClientConfigPath returns the path of the client configuration directory
+func getClientConfigPath() (string, error) {
+	path, err := config.GetConfigFilePath()
+	if err != nil {
+		return "", err
+	}
+	name := filepath.Base(path)
+	if name[0] == '.' {
+		name = name[1:]
+	}
+	path = filepath.Dir(path)
+	path = filepath.Join(path, ".scw")
+	path = filepath.Join(path, name)
+	return path, nil
+}
+
+// s3configure will prompt for accessKey and secretKey of the user and
+// save it for ams and par regions
+func s3Configure() error {
+	path, err := getClientConfigPath()
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(path, 0600)
+	if err != nil {
+		return err
+	}
+	var config clientConfig
+	configPath := filepath.Join(path, "config.json")
+	file, err := ioutil.ReadFile(configPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+		// Set default config values
+		config.Version = "9"
+		config.Hosts = make(map[string]clientHost)
+	} else {
+		// Retrieve config file
+		err = json.Unmarshal(file, &config)
+		if err != nil {
+			return err
+		}
+	}
+	// Get credentials from client
+	var accessKey string
+	var secretKey string
+	fmt.Print("AccessKey: ")
+	fmt.Scanln(&accessKey)
+	fmt.Print("SecretKey: ")
+	fmt.Scanln(&secretKey)
+	ams := config.Hosts["ams"]
+	par := config.Hosts["par"]
+	ams.AccessKey = accessKey
+	par.AccessKey = accessKey
+	ams.SecretKey = secretKey
+	par.SecretKey = secretKey
+	ams.Url = "https://s3.nl-ams.scw.cloud"
+	par.Url = "https://s3.fr-par.scw.cloud"
+	ams.Lookup = "auto"
+	par.Lookup = "auto"
+	ams.Api = "S3v2"
+	par.Api = "S3v2"
+	config.Hosts["ams"] = ams
+	config.Hosts["par"] = par
+	configFile, err := os.OpenFile(configPath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
+	if err != nil {
+		return err
+	}
+	defer configFile.Close()
+	encoder := json.NewEncoder(configFile)
+	encoder.SetIndent("", "\t")
+	err = encoder.Encode(config)
+	if err != nil {
+		return err
+	}
+	// fmt.Printf("%s\n", config)
+	return nil
+}
+
+// removeClientEnv will remove any environment variable related to the client
+func removeClientEnv() {
+	envs := os.Environ()
+	for i := 0; i < len(envs); i++ {
+		if strings.HasPrefix(envs[i], "MC_") {
+			os.Unsetenv(strings.Split(envs[i], "=")[0])
+		}
+	}
+}
+
+// S3 is the handler for 'scw s3'
+func S3(ctx CommandContext, args S3Args) error {
+	// Client own the config command, but we override it to configure Only
+	// the host we want
+	if len(args.Command) > 0 && args.Command[0] == "config" {
+		return s3Configure()
+	}
+	// Remove any client configuration environment variable
+	removeClientEnv()
+	// Get the scw cli configuration path
+	confPath, err := getClientConfigPath()
+	if err != nil {
+		return err
+	}
+	newArgs := make([]string, 0)
+	newArgs = append(newArgs, os.Args[:1]...)
+	// Override the client config directory
+	newArgs = append(newArgs, "--config-folder")
+	newArgs = append(newArgs, confPath)
+	newArgs = append(newArgs, "--quiet")
+	// Remove any scw cli options
+	newArgs = append(newArgs, args.Command...)
+	os.Args = newArgs
+	// Override args[0] to display `scw s3` in the client help pages
+	os.Args[0] = os.Args[0] + " s3"
+	minio.Main()
+	return nil
+}

pkg/config/config.go

@@ -57,6 +57,10 @@ func (c *Config) Save(configPath string) error {
 func GetConfig(scwrcPath string) (*Config, error) {
 	var err error
 
+	if scwrcPath != "" {
+		os.Setenv("SCW_CONFIG_PATH", scwrcPath)
+	}
+
 	orgid := os.Getenv("SCW_ORGANIZATION")
 	token := os.Getenv("SCW_TOKEN")
 	if token != "" && orgid != "" {

pkg/config/config_test.go

@@ -18,6 +18,7 @@ import (
 
 func TestGetConfigFilePath(t *testing.T) {
 	Convey("Testing GetConfigFilePath()", t, func() {
+		os.Unsetenv("SCW_CONFIG_PATH")
 		configPath, err := GetConfigFilePath()
 		So(err, ShouldBeNil)
 		So(configPath, ShouldNotEqual, "")

pkg/utils/utils.go

@@ -85,14 +85,14 @@ func NewSSHExecCmd(publicIPAddress, privateIPAddress, user string, port int, all
 	quiet := os.Getenv("DEBUG") != "1"
 	secureExec := os.Getenv("SCW_SECURE_EXEC") == "1"
 	sshCommand := &sshcommand.Command{
-		AllocateTTY:         allocateTTY,
-		Command:             command,
-		Host:                publicIPAddress,
-		Quiet:               quiet,
-		SkipHostKeyChecking: !secureExec,
-		User:                user,
-		NoEscapeCommand:     true,
-		Port:                port,
+		AllocateTTY:            allocateTTY,
+		Command:                command,
+		Host:                   publicIPAddress,
+		Quiet:                  quiet,
+		SkipHostKeyChecking:    !secureExec,
+		User:                   user,
+		NoEscapeCommand:        true,
+		Port:                   port,
 		EnableSSHKeyForwarding: enableSSHKeyForwarding,
 	}
 	if gatewayIPAddress != "" {