diff --git a/go/paddlecloud/simplefile.go b/go/paddlecloud/simplefile.go index 6ae45f6f..9ba499a5 100644 --- a/go/paddlecloud/simplefile.go +++ b/go/paddlecloud/simplefile.go @@ -10,6 +10,8 @@ import ( "net/url" "os" "path" + "path/filepath" + "strings" "github.com/PaddlePaddle/cloud/go/utils/restclient" "github.com/google/subcommands" @@ -45,7 +47,7 @@ func (p *SimpleFileCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfa } switch f.Arg(0) { case "put": - err := putFile(f.Arg(1), f.Arg(2)) + err := putFiles(f.Arg(1), f.Arg(2)) if err != nil { fmt.Fprintf(os.Stderr, "put file error: %s\n", err) return subcommands.ExitFailure @@ -87,13 +89,39 @@ func lsFile(dst string) error { return errors.New("list file error: " + errMsg) } items := respObj.(map[string]interface{})["items"].([]interface{}) - for _, i := range items { - fmt.Println(i.(string)) + for _, fn := range items { + fmt.Println(fn.(string)) + } + return nil +} + +func putFiles(src string, dest string) error { + f, err := os.Stat(src) + if err != nil { + return err + } + switch mode := f.Mode(); { + case mode.IsDir(): + if err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if info.Mode().IsRegular() { + srcs := strings.Split(filepath.Clean(src), string(os.PathSeparator)) + paths := strings.Split(path, string(os.PathSeparator)) + destFile := filepath.Join(dest, strings.Join(paths[len(srcs)-1:len(paths)], string(os.PathSeparator))) + putFile(path, destFile) + } + return nil + }); err != nil { + return err + } + + case mode.IsRegular(): + return putFile(src, dest) } return nil } func putFile(src string, dest string) error { + fmt.Printf("Uploading ... %s %s\n", src, dest) query := url.Values{} _, srcFile := path.Split(src) destDir, destFile := path.Split(dest) diff --git a/go/paddlecloud/submit.go b/go/paddlecloud/submit.go index bdad2cdc..fa62cf83 100644 --- a/go/paddlecloud/submit.go +++ b/go/paddlecloud/submit.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path" - "path/filepath" "github.com/PaddlePaddle/cloud/go/utils/config" "github.com/PaddlePaddle/cloud/go/utils/restclient" @@ -108,19 +107,8 @@ func (s *Submitter) Submit(jobPackage string, jobName string) error { // if jobPackage is not a local dir, skip uploading package. _, pkgerr := os.Stat(jobPackage) if pkgerr == nil { - // 1. upload user job package to pfs. - err := filepath.Walk(jobPackage, func(filePath string, info os.FileInfo, err error) error { - if info.IsDir() { - return nil - } - glog.V(10).Infof("Uploading %s...\n", filePath) - dest := path.Join("/pfs", Config.ActiveConfig.Name, "home", Config.ActiveConfig.Username, "jobs", jobName, filepath.Base(filePath)) - fmt.Printf("uploading: %s...\n", filePath) - return putFile(filePath, dest) - }) - if err != nil { - return err - } + dest := path.Join("/pfs", Config.ActiveConfig.Name, "home", Config.ActiveConfig.Username, "jobs", jobName) + return putFiles(jobPackage, dest) } else if os.IsNotExist(pkgerr) { glog.Warning("jobpackage not a local dir, skip upload.") }