Fix notebook sync dir and dataset loading (#227)

Fixes #226
Fixes #229

examples/notebook/notebook.yaml

@@ -4,8 +4,8 @@ metadata:
   name: example
 spec:
   image: substratusai/base:latest
-  model:
-    name: facebook-opt-125m
   params:
     abc: xyz
     x: 123
+  #model:
+  #  name: facebook-opt-125m

kubectl/internal/client/client.go

@@ -33,7 +33,7 @@ var _ Interface = &Client{}
 type Interface interface {
 	PortForwardNotebook(ctx context.Context, verbose bool, nb *apiv1.Notebook, ready chan struct{}) error
 	Resource(obj Object) (*Resource, error)
-	SyncFilesFromNotebook(context.Context, *apiv1.Notebook) error
+	SyncFilesFromNotebook(context.Context, *apiv1.Notebook, string) error
 }
 
 func NewClient(inf kubernetes.Interface, cfg *rest.Config) Interface {
@@ -79,14 +79,14 @@ func (c *Client) Resource(obj Object) (*Resource, error) {
 
 	helper := resource.NewHelper(restClient, mapping)
 	helper.FieldManager = FieldManager
-	//helper.FieldValidation = "Strict"
+	// helper.FieldValidation = "Strict"
 
 	// Use the REST helper to create the object in the "default" namespace.
 	return &Resource{Helper: helper}, nil
 }
 
 func newRestClient(restConfig *rest.Config, gv schema.GroupVersion) (rest.Interface, error) {
-	//restConfig.ContentConfig = resource.UnstructuredPlusDefaultContentConfig()
+	// restConfig.ContentConfig = resource.UnstructuredPlusDefaultContentConfig()
 	restConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
 	restConfig.GroupVersion = &gv
 	if len(gv.Group) == 0 {

kubectl/internal/client/sync.go

@@ -14,17 +14,18 @@ import (
 	"strings"
 	"sync"
 
-	apiv1 "github.com/substratusai/substratus/api/v1"
-	"github.com/substratusai/substratus/kubectl/internal/cp"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
 	"k8s.io/klog/v2"
+
+	apiv1 "github.com/substratusai/substratus/api/v1"
+	"github.com/substratusai/substratus/kubectl/internal/cp"
 )
 
-func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook) error {
+func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook, localDir string) error {
 	podRef := podForNotebook(nb)
 	const containerName = "notebook"
 
@@ -88,8 +89,7 @@ func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook)
 				continue
 			}
 
-			localDir := "src"
-			localPath := filepath.Join(localDir, relPath)
+			localPath := filepath.Join(localDir, "src", relPath)
 
 			// Possible: CREATE, REMOVE, WRITE, RENAME, CHMOD
 			if event.Op == "WRITE" || event.Op == "CREATE" {
@@ -125,7 +125,8 @@ func (c *Client) SyncFilesFromNotebook(ctx context.Context, nb *apiv1.Notebook)
 }
 
 func (c *Client) exec(ctx context.Context, podRef types.NamespacedName,
-	command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+	command string, stdin io.Reader, stdout io.Writer, stderr io.Writer,
+) error {
 	cmd := []string{
 		"sh",
 		"-c",
@@ -134,11 +135,12 @@ func (c *Client) exec(ctx context.Context, podRef types.NamespacedName,
 	req := c.Interface.CoreV1().RESTClient().Post().Resource("pods").Name(podRef.Name).
 		Namespace(podRef.Namespace).SubResource("exec")
 	option := &corev1.PodExecOptions{
-		Command: cmd,
-		Stdin:   true,
-		Stdout:  true,
-		Stderr:  true,
-		TTY:     true,
+		Command:   cmd,
+		Stdin:     true,
+		Stdout:    true,
+		Stderr:    true,
+		TTY:       true,
+		Container: "notebook",
 	}
 	if stdin == nil {
 		option.Stdin = false

kubectl/internal/commands/notebook.go

@@ -239,7 +239,7 @@ func Notebook() *cobra.Command {
 						// Stop other goroutines.
 						cancel()
 					}()
-					if err := c.SyncFilesFromNotebook(ctx, nb); err != nil {
+					if err := c.SyncFilesFromNotebook(ctx, nb, flags.build); err != nil {
 						if !errors.Is(err, context.Canceled) {
 							klog.Errorf("Error syncing files from notebook: %v", err)
 						}