Support for configurations consisting of multiple files

README.md

@@ -32,8 +32,10 @@ After configuration, it is possible to:
 
 - `list` apps in project
 - `describe` apps to get their status
-- `download` to get application's training data file
-- `upload` to upload a new version of the training file and start training
+- `download` to get application's training data files as tar
+- `upload` to upload a new package of training files and start training
+
+The versioning of the configuration files should be done properly, ie. keep them in a version control system. Consider the upload/download functionality to be a tool for the training pipeline instead of collaboration or versioning.
 
 # Develop and debug the tool
 

cmd/config.go

@@ -53,7 +53,7 @@ var configAddCmd = &cobra.Command{
 }
 
 var configRemoveCmd = &cobra.Command{
-	Use: "remove",
+	Use:   "remove",
 	Short: "Remove a context from configuration",
 	PreRunE: func(cmd *cobra.Command, args []string) error {
 		name, _ := cmd.Flags().GetString("name")
@@ -82,7 +82,7 @@ var configRemoveCmd = &cobra.Command{
 }
 
 var configUseCmd = &cobra.Command{
-	Use: "use",
+	Use:   "use",
 	Short: "Select the default context used",
 	PreRunE: func(cmd *cobra.Command, args []string) error {
 		name, _ := cmd.Flags().GetString("name")

cmd/root.go

@@ -5,6 +5,8 @@ import (
 	"crypto/tls"
 	"fmt"
 	"log"
+	"os"
+	"path/filepath"
 	"strings"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -51,16 +53,27 @@ func initConfig() {
 	viper.AddConfigPath(home)
 	viper.AddConfigPath(".")
 	viper.SetConfigName(".speechly")
+	viper.SetConfigType("yaml")
 	viper.AutomaticEnv()
 
 	if err := viper.ReadInConfig(); err != nil {
-		log.Println("Please create a configuration file first:")
-		log.Println("")
-		log.Println("\tspeechly config add --apikey APIKEY")
-		log.Println("")
-	}
-	if err := viper.Unmarshal(&conf); err != nil {
-		log.Fatalf("Failed to unmarshal config file %s: %s", viper.ConfigFileUsed(), err)
+		if os.Args[1] != "config" {
+			log.Println("Please create a configuration file first:")
+			log.Println("")
+			log.Println("\tspeechly config add --apikey APIKEY")
+			log.Println("")
+			os.Exit(1)
+		}
+		// viper has a problem with non-existent config files, just touch the default:
+		file, err := os.Create(filepath.Join(home, ".speechly.yaml"))
+		if err != nil {
+			log.Fatalf("Could not initialize speechly config file: %s", err)
+		}
+		file.Close()
+	} else {
+		if err := viper.Unmarshal(&conf); err != nil {
+			log.Fatalf("Failed to unmarshal config file %s: %s", viper.ConfigFileUsed(), err)
+		}
 	}
 	for _, item := range conf.Contexts {
 		if item.Name == conf.CurrentContext {
@@ -77,8 +90,6 @@ func Execute() error {
 
 	md := metadata.Pairs("authorization", fmt.Sprintf("Bearer %s", sc.Apikey))
 	ctx := metadata.NewOutgoingContext(context.Background(), md)
-	//ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
-	//defer cancel()
 
 	serverAddr := sc.Host
 	opts := []grpc.DialOption{

cmd/upload.go

@@ -1,64 +1,113 @@
 package cmd
 
 import (
-	"io"
+	"archive/tar"
+	"bytes"
+	"io/ioutil"
 	"log"
-	"os"
+	"path/filepath"
+	"regexp"
 
 	"github.com/spf13/cobra"
 
 	configv1 "github.com/speechly/cli/gen/go/speechly/config/v1"
 )
 
+type UploadWriter struct {
+	appId  string
+	stream configv1.ConfigAPI_UploadTrainingDataClient
+}
+
+func (u UploadWriter) Write(data []byte) (n int, err error) {
+	req := &configv1.UploadTrainingDataRequest{AppId: u.appId, DataChunk: data, ContentType: configv1.UploadTrainingDataRequest_CONTENT_TYPE_TAR}
+	if err = u.stream.Send(req); err != nil {
+		return 0, err
+	}
+	return len(data), nil
+}
+
 var uploadCmd = &cobra.Command{
 	Use:   "upload",
 	Short: "Send a training data and configuration yaml to training",
-	Args:  cobra.NoArgs,
+	Args:  cobra.MaximumNArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
 		ctx := cmd.Context()
 		appId, _ := cmd.Flags().GetString("app")
-		inFile, _ := cmd.Flags().GetString("file")
-		contentTypeInt, _ := cmd.Flags().GetInt("content_type")
-		contentType := configv1.UploadTrainingDataRequest_ContentType(contentTypeInt)
-
-		reader, err := os.Open(inFile)
-		if err != nil {
-			log.Fatalf("Could not open file: %s: %s", inFile, err)
+		var inDir string
+		if len(args) > 0 {
+			inDir = args[0]
+		} else {
+			inDir = "./"
+		}
+		absPath, _ := filepath.Abs(inDir)
+		log.Printf("Project dir: %s\n", absPath)
+		// create a tar package from files in memory
+		buf := createTarFromDir(inDir)
+		if buf.Len() == 0 {
+			log.Fatalf("Nothing to upload.")
 		}
-		defer reader.Close()
 
+		// open a stream for upload
 		stream, err := client.UploadTrainingData(ctx)
+		if err != nil {
+			log.Fatalf("Failed to open upload stream: %s", err)
+		}
 
-		buffer := make([]byte, 32768)
-		total := 0
-		for {
-			n, err := reader.Read(buffer)
-			if err == io.EOF {
-				break
-			}
-			if err != nil {
-				log.Fatalf("Could not read file %s: %s", inFile, err)
-			}
-			total += n
-			req := &configv1.UploadTrainingDataRequest{AppId: appId, DataChunk: buffer[:n], ContentType: contentType}
-			if err = stream.Send(req); err != nil {
-				log.Fatalf("Uploading training data failed: %s", err)
-			}
+		// flush the tar from memory to the stream
+		uploadWriter := UploadWriter{appId, stream}
+		n, err := buf.WriteTo(uploadWriter)
+		if err != nil {
+			log.Fatalf("Streaming file data failed: %s", err)
 		}
+
 		// Response from upload is empty, ignore:
 		_, err = stream.CloseAndRecv()
 		if err != nil {
 			log.Fatalf("Upload failed: %s", err)
 		}
-		cmd.Printf("File %s (%d bytes) uploaded\n", inFile, total)
+
+		cmd.Printf("%d bytes uploaded", n)
 	},
 }
 
+func createTarFromDir(inDir string) bytes.Buffer {
+	files, err := ioutil.ReadDir(inDir)
+	if err != nil {
+		log.Fatalf("Could not read files from %s", inDir)
+	}
+	// only accept yaml and csv files in the tar package
+	configFileMatch := regexp.MustCompile(`.*?(csv|yaml)$`)
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	for _, f := range files {
+		if configFileMatch.MatchString(f.Name()) {
+			log.Printf("Adding %s (%d bytes)\n", f.Name(), f.Size())
+			hdr := &tar.Header{
+				Name: f.Name(),
+				Mode: 0600,
+				Size: f.Size(),
+			}
+			if err := tw.WriteHeader(hdr); err != nil {
+				log.Fatalf("Failed to create a tar header: %s", err)
+			}
+
+			contents, err := ioutil.ReadFile(filepath.Join(inDir, f.Name()))
+			if err != nil {
+				log.Fatalf("Failed to read file: %s", err)
+			}
+			if _, err := tw.Write(contents); err != nil {
+				log.Fatalf("Failed to tar file: %s", err)
+			}
+		}
+	}
+	if err := tw.Close(); err != nil {
+		log.Fatalf("Package finalization failed: %s", err)
+	}
+	return buf
+}
+
 func init() {
 	rootCmd.AddCommand(uploadCmd)
 	uploadCmd.Flags().StringP("app", "a", "", "application id to upload the files to.")
 	uploadCmd.MarkFlagRequired("app")
-	uploadCmd.Flags().StringP("file", "f", "", "File to upload. Will start training.")
-	uploadCmd.MarkFlagRequired("file")
-	uploadCmd.Flags().Int("content_type", 0, "Content type of the training data. 1 for .yaml, 2 for .tar.")
 }

go.mod

@@ -3,15 +3,24 @@ module github.com/speechly/cli
 go 1.14
 
 require (
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/golang/protobuf v1.4.2
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/mitchellh/mapstructure v1.3.3 // indirect
+	github.com/pelletier/go-toml v1.8.0 // indirect
+	github.com/spf13/afero v1.3.3 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.4.0
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.4.0 // indirect
-	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
-	golang.org/x/sys v0.0.0-20190412213103-97732733099d // indirect
-	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.24.0
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
+	golang.org/x/sys v0.0.0-20200805065543-0cf7623e9dbd // indirect
+	golang.org/x/text v0.3.3 // indirect
+	google.golang.org/genproto v0.0.0-20200804151602-45615f50871c // indirect
+	google.golang.org/grpc v1.31.0
+	google.golang.org/protobuf v1.25.0
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	gopkg.in/yaml.v2 v2.2.4 // indirect
+	gopkg.in/ini.v1 v1.57.0 // indirect
 )