feat(attach,upload): add destination-dir flag for custom file paths

cmd/attach.go

@@ -51,6 +51,7 @@ func init() {
 	flags := attachCmd.Flags()
 	flags.StringVarP(&attachConfig.Source, "source", "s", "", "source model artifact name")
 	flags.StringVarP(&attachConfig.Target, "target", "t", "", "target model artifact name")
+	flags.StringVarP(&attachConfig.DestinationDir, "destination-dir", "d", "", "destination directory for the attached file should be specified as a relative path; by default, it will match the original directory of the attachment")
 	flags.BoolVarP(&attachConfig.OutputRemote, "output-remote", "", false, "turning on this flag will output model artifact to remote registry directly")
 	flags.BoolVarP(&attachConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
 	flags.BoolVarP(&attachConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")

cmd/upload.go

@@ -53,6 +53,7 @@ func init() {
 	flags.BoolVarP(&uploadConfig.PlainHTTP, "plain-http", "", false, "turning on this flag will use plain HTTP instead of HTTPS")
 	flags.BoolVarP(&uploadConfig.Insecure, "insecure", "", false, "turning on this flag will disable TLS verification")
 	flags.BoolVar(&uploadConfig.Raw, "raw", true, "turning on this flag will upload model artifact layer in raw format")
+	flags.StringVar(&uploadConfig.DestinationDir, "destination-dir", "", "destination directory for the uploaded file should be specified as a relative path; by default, it will match the original directory of the uploaded file")
 
 	if err := viper.BindPFlags(flags); err != nil {
 		panic(fmt.Errorf("bind cache list flags to viper: %w", err))

pkg/backend/attach.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	pathfilepath "path/filepath"
 	"reflect"
 	"slices"
 	"sort"
@@ -74,7 +75,7 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 
 	logrus.Infof("attach: loaded source model config [%+v]", srcModelConfig)
 
-	proc := b.getProcessor(filepath, cfg.Raw)
+	proc := b.getProcessor(cfg.DestinationDir, filepath, cfg.Raw)
 	if proc == nil {
 		return fmt.Errorf("failed to get processor for file %s", filepath)
 	}
@@ -88,15 +89,20 @@ func (b *backend) Attach(ctx context.Context, filepath string, cfg *config.Attac
 	pb.Start()
 	defer pb.Stop()
 
+	destPath := filepath
+	if cfg.DestinationDir != "" {
+		destPath = pathfilepath.Join(cfg.DestinationDir, pathfilepath.Base(filepath))
+	}
+
 	layers := srcManifest.Layers
 	// If attach a normal file, we need to process it and create a new layer.
 	if !cfg.Config {
 		var foundLayer *ocispec.Descriptor
 		for _, layer := range srcManifest.Layers {
 			if anno := layer.Annotations; anno != nil {
-				if anno[modelspec.AnnotationFilepath] == filepath || anno[legacymodelspec.AnnotationFilepath] == filepath {
+				if anno[modelspec.AnnotationFilepath] == destPath || anno[legacymodelspec.AnnotationFilepath] == destPath {
 					if !cfg.Force {
-						return fmt.Errorf("file %s already exists, please use --force to overwrite if you want to attach it forcibly", filepath)
+						return fmt.Errorf("file %s already exists, please use --force to overwrite if you want to attach it forcibly", destPath)
 					}
 
 					foundLayer = &layer
@@ -299,37 +305,37 @@ func (b *backend) getModelConfig(ctx context.Context, reference string, desc oci
 	return &model, nil
 }
 
-func (b *backend) getProcessor(filepath string, rawMediaType bool) processor.Processor {
+func (b *backend) getProcessor(destDir, filepath string, rawMediaType bool) processor.Processor {
 	if modelfile.IsFileType(filepath, modelfile.ConfigFilePatterns) {
 		mediaType := legacymodelspec.MediaTypeModelWeightConfig
 		if rawMediaType {
 			mediaType = legacymodelspec.MediaTypeModelWeightConfigRaw
 		}
-		return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath})
+		return processor.NewModelConfigProcessor(b.store, mediaType, []string{filepath}, destDir)
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.ModelFilePatterns) {
 		mediaType := legacymodelspec.MediaTypeModelWeight
 		if rawMediaType {
 			mediaType = legacymodelspec.MediaTypeModelWeightRaw
 		}
-		return processor.NewModelProcessor(b.store, mediaType, []string{filepath})
+		return processor.NewModelProcessor(b.store, mediaType, []string{filepath}, destDir)
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.CodeFilePatterns) {
 		mediaType := legacymodelspec.MediaTypeModelCode
 		if rawMediaType {
 			mediaType = legacymodelspec.MediaTypeModelCodeRaw
 		}
-		return processor.NewCodeProcessor(b.store, mediaType, []string{filepath})
+		return processor.NewCodeProcessor(b.store, mediaType, []string{filepath}, destDir)
 	}
 
 	if modelfile.IsFileType(filepath, modelfile.DocFilePatterns) {
 		mediaType := legacymodelspec.MediaTypeModelDoc
 		if rawMediaType {
 			mediaType = legacymodelspec.MediaTypeModelDocRaw
 		}
-		return processor.NewDocProcessor(b.store, mediaType, []string{filepath})
+		return processor.NewDocProcessor(b.store, mediaType, []string{filepath}, destDir)
 	}
 
 	return nil

pkg/backend/attach_test.go

@@ -73,7 +73,7 @@ func TestGetProcessor(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.filepath, func(t *testing.T) {
-			proc := b.getProcessor(tt.filepath, false)
+			proc := b.getProcessor("", tt.filepath, false)
 			if tt.wantType == "" {
 				assert.Nil(t, proc)
 			} else {

pkg/backend/build.go

@@ -170,31 +170,31 @@ func (b *backend) getProcessors(modelfile modelfile.Modelfile, cfg *config.Build
 		if cfg.Raw {
 			mediaType = modelspec.MediaTypeModelWeightConfigRaw
 		}
-		processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs))
+		processors = append(processors, processor.NewModelConfigProcessor(b.store, mediaType, configs, ""))
 	}
 
 	if models := modelfile.GetModels(); len(models) > 0 {
 		mediaType := modelspec.MediaTypeModelWeight
 		if cfg.Raw {
 			mediaType = modelspec.MediaTypeModelWeightRaw
 		}
-		processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models))
+		processors = append(processors, processor.NewModelProcessor(b.store, mediaType, models, ""))
 	}
 
 	if codes := modelfile.GetCodes(); len(codes) > 0 {
 		mediaType := modelspec.MediaTypeModelCode
 		if cfg.Raw {
 			mediaType = modelspec.MediaTypeModelCodeRaw
 		}
-		processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes))
+		processors = append(processors, processor.NewCodeProcessor(b.store, mediaType, codes, ""))
 	}
 
 	if docs := modelfile.GetDocs(); len(docs) > 0 {
 		mediaType := modelspec.MediaTypeModelDoc
 		if cfg.Raw {
 			mediaType = modelspec.MediaTypeModelDocRaw
 		}
-		processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs))
+		processors = append(processors, processor.NewDocProcessor(b.store, mediaType, docs, ""))
 	}
 
 	return processors

pkg/backend/build/builder.go

@@ -58,7 +58,7 @@ const (
 // Builder is an interface for building artifacts.
 type Builder interface {
 	// BuildLayer builds the layer blob from the given file path.
-	BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error)
+	BuildLayer(ctx context.Context, mediaType, workDir, path, destPath string, hooks hooks.Hooks) (ocispec.Descriptor, error)
 
 	// BuildConfig builds the config blob of the artifact.
 	BuildConfig(ctx context.Context, config modelspec.Model, hooks hooks.Hooks) (ocispec.Descriptor, error)
@@ -69,7 +69,7 @@ type Builder interface {
 
 type OutputStrategy interface {
 	// OutputLayer outputs the layer blob to the storage (local or remote).
-	OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error)
+	OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error)
 
 	// OutputConfig outputs the config blob to the storage (local or remote).
 	OutputConfig(ctx context.Context, mediaType, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error)
@@ -122,7 +122,7 @@ type abstractBuilder struct {
 	interceptor interceptor.Interceptor
 }
 
-func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path string, hooks hooks.Hooks) (ocispec.Descriptor, error) {
+func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, path, destPath string, hooks hooks.Hooks) (ocispec.Descriptor, error) {
 	info, err := os.Stat(path)
 	if err != nil {
 		return ocispec.Descriptor{}, fmt.Errorf("failed to get file info: %w", err)
@@ -179,7 +179,7 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
 		}()
 	}
 
-	desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, digest, size, reader, hooks)
+	desc, err := ab.strategy.OutputLayer(ctx, mediaType, relPath, destPath, digest, size, reader, hooks)
 	if err != nil {
 		return desc, err
 	}

pkg/backend/build/builder_test.go

@@ -123,23 +123,23 @@ func (s *BuilderTestSuite) TestBuildLayer() {
 			Size:      100,
 		}
 
-		s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything).
+		s.mockOutputStrategy.On("OutputLayer", mock.Anything, "test/media-type.tar", "test-file.txt", "", mock.AnythingOfType("string"), mock.AnythingOfType("int64"), mock.AnythingOfType("*io.PipeReader"), mock.Anything).
 			Return(expectedDesc, nil)
 
-		desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, hooks.NewHooks())
+		desc, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempFile, "", hooks.NewHooks())
 		s.NoError(err)
 		s.Equal(expectedDesc.MediaType, desc.MediaType)
 		s.Equal(expectedDesc.Digest, desc.Digest)
 		s.Equal(expectedDesc.Size, desc.Size)
 	})
 
 	s.Run("file not found", func() {
-		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), hooks.NewHooks())
+		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, filepath.Join(s.tempDir, "non-existent.txt"), "", hooks.NewHooks())
 		s.Error(err)
 	})
 
 	s.Run("directory not supported", func() {
-		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, hooks.NewHooks())
+		_, err := s.builder.BuildLayer(context.Background(), "test/media-type.tar", s.tempDir, s.tempDir, "", hooks.NewHooks())
 		s.Error(err)
 		s.True(strings.Contains(err.Error(), "is a directory and not supported yet"))
 	})

pkg/backend/build/local.go

@@ -46,20 +46,24 @@ type localOutput struct {
 }
 
 // OutputLayer outputs the layer blob to the local storage.
-func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
+func (lo *localOutput) OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
 	reader = hooks.OnStart(relPath, size, reader)
 	digest, size, err := lo.store.PushBlob(ctx, lo.repo, reader, ocispec.Descriptor{})
 	if err != nil {
 		hooks.OnError(relPath, err)
 		return ocispec.Descriptor{}, fmt.Errorf("failed to push blob to storage: %w", err)
 	}
 
+	if destPath == "" {
+		destPath = relPath
+	}
+
 	desc := ocispec.Descriptor{
 		MediaType: mediaType,
 		Digest:    godigest.Digest(digest),
 		Size:      size,
 		Annotations: map[string]string{
-			modelspec.AnnotationFilepath: relPath,
+			modelspec.AnnotationFilepath: destPath,
 		},
 	}
 

pkg/backend/build/local_test.go

@@ -72,7 +72,7 @@ func (s *LocalOutputTestSuite) TestOutputLayer() {
 		s.mockStorage.On("PushBlob", s.ctx, "test-repo", mock.Anything, ocispec.Descriptor{}).
 			Return(expectedDigest, expectedSize, nil).Once()
 
-		desc, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "test-file.txt", expectedDigest, expectedSize, reader, hooks.NewHooks())
+		desc, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "test-file.txt", "", expectedDigest, expectedSize, reader, hooks.NewHooks())
 
 		s.NoError(err)
 		s.Equal("test/mediatype", desc.MediaType)
@@ -88,7 +88,7 @@ func (s *LocalOutputTestSuite) TestOutputLayer() {
 		s.mockStorage.On("PushBlob", s.ctx, "test-repo", mock.Anything, ocispec.Descriptor{}).
 			Return("", int64(0), errors.New("storage error")).Once()
 
-		_, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "/work", "test-file.txt", int64(0), reader, hooks.NewHooks())
+		_, err := s.localOutput.OutputLayer(s.ctx, "test/mediatype", "/work", "test-file.txt", "", int64(0), reader, hooks.NewHooks())
 
 		s.Error(err)
 		s.Contains(err.Error(), "failed to push blob to storage")

pkg/backend/build/remote.go

@@ -51,13 +51,17 @@ type remoteOutput struct {
 }
 
 // OutputLayer outputs the layer blob to the remote storage.
-func (ro *remoteOutput) OutputLayer(ctx context.Context, mediaType, relPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
+func (ro *remoteOutput) OutputLayer(ctx context.Context, mediaType, relPath, destPath, digest string, size int64, reader io.Reader, hooks hooks.Hooks) (ocispec.Descriptor, error) {
+	if destPath == "" {
+		destPath = relPath
+	}
+
 	desc := ocispec.Descriptor{
 		MediaType: mediaType,
 		Digest:    godigest.Digest(digest),
 		Size:      size,
 		Annotations: map[string]string{
-			modelspec.AnnotationFilepath: relPath,
+			modelspec.AnnotationFilepath: destPath,
 		},
 	}
 

pkg/backend/processor/base.go

@@ -48,6 +48,10 @@ type base struct {
 	mediaType string
 	// patterns is the list of patterns to match.
 	patterns []string
+	// destDir is the destination dir for the processed content,
+	// which is used to store in the layer filepath annotation,
+	// it can be empty and by default is relative path to the workDir.
+	destDir string
 }
 
 // Process implements the Processor interface, which can be reused by other processors.
@@ -140,7 +144,12 @@ func (b *base) Process(ctx context.Context, builder build.Builder, workDir strin
 			if err := retry.Do(func() error {
 				logrus.Debugf("processor: processing %s file %s", b.name, path)
 
-				desc, err := builder.BuildLayer(ctx, b.mediaType, workDir, path, hooks.NewHooks(
+				var destPath string
+				if b.destDir != "" {
+					destPath = filepath.Join(b.destDir, filepath.Base(path))
+				}
+
+				desc, err := builder.BuildLayer(ctx, b.mediaType, workDir, path, destPath, hooks.NewHooks(
 					hooks.WithOnStart(func(name string, size int64, reader io.Reader) io.Reader {
 						return tracker.Add(internalpb.NormalizePrompt("Building layer"), name, size, reader)
 					}),

pkg/backend/processor/code.go

@@ -30,13 +30,14 @@ const (
 )
 
 // NewCodeProcessor creates a new code processor.
-func NewCodeProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+func NewCodeProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor {
 	return &codeProcessor{
 		base: &base{
 			name:      codeProcessorName,
 			store:     store,
 			mediaType: mediaType,
 			patterns:  patterns,
+			destDir:   destDir,
 		},
 	}
 }

pkg/backend/processor/code_test.go

@@ -44,7 +44,7 @@ type codeProcessorSuite struct {
 func (s *codeProcessorSuite) SetupTest() {
 	s.mockStore = &storage.Storage{}
 	s.mockBuilder = &buildmock.Builder{}
-	s.processor = NewCodeProcessor(s.mockStore, modelspec.MediaTypeModelCode, []string{"*.py"})
+	s.processor = NewCodeProcessor(s.mockStore, modelspec.MediaTypeModelCode, []string{"*.py"}, "")
 	// generate test files for prorcess.
 	s.workDir = s.Suite.T().TempDir()
 	if err := os.WriteFile(filepath.Join(s.workDir, "test.py"), []byte(""), 0644); err != nil {
@@ -58,7 +58,7 @@ func (s *codeProcessorSuite) TestName() {
 
 func (s *codeProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
+	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
 		Digest: godigest.Digest("sha256:1234567890abcdef"),
 		Size:   int64(1024),
 		Annotations: map[string]string{

pkg/backend/processor/doc.go

@@ -30,13 +30,14 @@ const (
 )
 
 // NewDocProcessor creates a new doc processor.
-func NewDocProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+func NewDocProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor {
 	return &docProcessor{
 		base: &base{
 			name:      docProcessorName,
 			store:     store,
 			mediaType: mediaType,
 			patterns:  patterns,
+			destDir:   destDir,
 		},
 	}
 }

pkg/backend/processor/doc_test.go

@@ -44,7 +44,7 @@ type docProcessorSuite struct {
 func (s *docProcessorSuite) SetupTest() {
 	s.mockStore = &storage.Storage{}
 	s.mockBuilder = &buildmock.Builder{}
-	s.processor = NewDocProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"})
+	s.processor = NewDocProcessor(s.mockStore, modelspec.MediaTypeModelDoc, []string{"LICENSE"}, "")
 	// generate test files for prorcess.
 	s.workDir = s.Suite.T().TempDir()
 	if err := os.WriteFile(filepath.Join(s.workDir, "LICENSE"), []byte(""), 0644); err != nil {
@@ -58,7 +58,7 @@ func (s *docProcessorSuite) TestName() {
 
 func (s *docProcessorSuite) TestProcess() {
 	ctx := context.Background()
-	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
+	s.mockBuilder.On("BuildLayer", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(ocispec.Descriptor{
 		Digest: godigest.Digest("sha256:1234567890abcdef"),
 		Size:   int64(1024),
 		Annotations: map[string]string{

pkg/backend/processor/model.go

@@ -30,13 +30,14 @@ const (
 )
 
 // NewModelProcessor creates a new model processor.
-func NewModelProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+func NewModelProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor {
 	return &modelProcessor{
 		base: &base{
 			name:      modelProcessorName,
 			store:     store,
 			mediaType: mediaType,
 			patterns:  patterns,
+			destDir:   destDir,
 		},
 	}
 }

pkg/backend/processor/model_config.go

@@ -30,13 +30,14 @@ const (
 )
 
 // NewModelConfigProcessor creates a new model config processor.
-func NewModelConfigProcessor(store storage.Storage, mediaType string, patterns []string) Processor {
+func NewModelConfigProcessor(store storage.Storage, mediaType string, patterns []string, destDir string) Processor {
 	return &modelConfigProcessor{
 		base: &base{
 			name:      modelConfigProcessorName,
 			store:     store,
 			mediaType: mediaType,
 			patterns:  patterns,
+			destDir:   destDir,
 		},
 	}
 }