Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: push and pull subcommand for model command #10483

Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 3 additions & 2 deletions go.mod
Expand Up @@ -60,7 +60,7 @@ require (
github.com/vektah/gqlparser/v2 v2.5.10
github.com/vmihailenco/taskq/v3 v3.2.9
golang.org/x/oauth2 v0.15.0
golang.org/x/sync v0.5.0
golang.org/x/sync v0.6.0
golang.org/x/text v0.14.0
gonum.org/v1/gonum v0.14.0
google.golang.org/api v0.152.0
Expand Down Expand Up @@ -255,7 +255,7 @@ require (
github.com/onsi/ginkgo/v2 v2.15.0 // indirect
github.com/open-policy-agent/opa v0.57.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
github.com/opencontainers/image-spec v1.1.0-rc6 // indirect
github.com/opencontainers/runc v1.1.12 // indirect
github.com/openshift/api v3.9.0+incompatible // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
Expand Down Expand Up @@ -320,6 +320,7 @@ require (
k8s.io/kubectl v0.28.4 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
oras.land/oras-go v1.2.4 // indirect
oras.land/oras-go/v2 v2.4.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Expand Up @@ -1843,6 +1843,8 @@ github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zM
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
github.com/opencontainers/image-spec v1.1.0-rc6 h1:XDqvyKsJEbRtATzkgItUqBA7QHk58yxX1Ov9HERHNqU=
github.com/opencontainers/image-spec v1.1.0-rc6/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss=
Expand Down Expand Up @@ -2464,6 +2466,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -3199,6 +3203,8 @@ modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8=
oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY=
oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg=
oras.land/oras-go/v2 v2.4.0 h1:i+Wt5oCaMHu99guBD0yuBjdLvX7Lz8ukPbwXdR7uBMs=
oras.land/oras-go/v2 v2.4.0/go.mod h1:osvtg0/ClRq1KkydMAEu/IxFieyjItcsQ4ut4PPF+f8=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/letsencrypt v0.0.3/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
Expand Down
138 changes: 136 additions & 2 deletions mesheryctl/internal/cli/root/system/model.go
Expand Up @@ -15,6 +15,7 @@
package system

import (
"context"
"encoding/json"
"fmt"
"io"
Expand All @@ -29,11 +30,17 @@ import (
"github.com/layer5io/meshery/server/models"
"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1"
"github.com/manifoldco/promptui"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
"oras.land/oras-go/v2/registry/remote/retry"
)

var (
Expand All @@ -47,6 +54,12 @@ var (

// Color for the whiteboard printer
whiteBoardPrinter = color.New(color.FgHiBlack, color.BgWhite, color.Bold)

username string
password string
registry string
repository string
tag string
)

// represents the mesheryctl exp model list subcommand.
Expand Down Expand Up @@ -413,6 +426,120 @@ mesheryctl exp model search [query-text]
},
}

// add command to push aritfacts to the oci registry
var pushModelCmd = &cobra.Command{
Use: "push",
Short: "push model",
Long: "push a model to the registry",
Example: `
// Push a model
mesheryctl exp model push [path-to-model] --username [username] --password [password] --registry [registry] --tag [tag] --repository [repository]
`,
// skip preRunE as it is not required for this command
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return cmd.Help()
}

// path to model
modelPath := args[0]
RipulHandoo marked this conversation as resolved.
Show resolved Hide resolved

// Push logic
fs, err := file.New(".")
RipulHandoo marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return err
}
ctx := context.Background()

mediaType := "application/vnd.test.folder"
RipulHandoo marked this conversation as resolved.
Show resolved Hide resolved
fileNames := []string{modelPath}
fileDescriptors := make([]v1.Descriptor, 0)
for _, name := range fileNames {
fileDescriptor, err := fs.Add(ctx, name, mediaType, "")
if err != nil {
panic(err)
}
fileDescriptors = append(fileDescriptors, fileDescriptor)
fmt.Printf("file descriptor for %s: %v\n", name, fileDescriptor)
RipulHandoo marked this conversation as resolved.
Show resolved Hide resolved
}
artifactType := "application/vnd.test.artifact"
opts := oras.PackManifestOptions{
Layers: fileDescriptors,
}
manifestDescriptor, err := oras.PackManifest(ctx, fs, oras.PackManifestVersion1_1_RC4, artifactType, opts)
if err != nil {
return err
}

tag := tag
if err = fs.Tag(ctx, manifestDescriptor, tag); err != nil {
return err
}

reg := registry
pathToFolder := fmt.Sprintf("%s/%s/%s", reg, username, repository)
repo, err := remote.NewRepository(pathToFolder)

if err != nil {
return err
}
repo.Client = &auth.Client{
Client: retry.DefaultClient,
Cache: auth.NewCache(),
Credential: auth.StaticCredential(reg, auth.Credential{
Username: username,
Password: password,
}),
}

_, err = oras.Copy(ctx, fs, tag, repo, tag, oras.DefaultCopyOptions)
if err != nil {
return err
}

fmt.Println("Succeed")
return nil
},
}

// add command to pull aritfacts from the oci registry
var pullModelCmd = &cobra.Command{
Use: "pull",
Short: "pull model",
Long: "pull a model to the registry",
Example: `
// Push a model
mesheryctl exp model pull --registry_address [registry] --tag [tag]
`,
// skip preRunE as it is not required for this command
RunE: func(cmd *cobra.Command, args []string) error {
// Push logic
fs, err := file.New("./tmp/")
if err != nil {
panic(err)
}
defer fs.Close()
// 1. Connect to a remote repository
ctx := context.Background()
reg := registry
pathToFolder := fmt.Sprintf("%s", reg)
repo, err := remote.NewRepository(pathToFolder)
if err != nil {
panic(err)
}

// 2. Copy from the remote repository to the OCI layout store
tag := tag
manifestDescriptor, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
if err != nil {
panic(err)
}

fmt.Println("manifest pulled:", manifestDescriptor.Digest, manifestDescriptor.MediaType)
return nil
},
}

// ModelCmd represents the mesheryctl exp model command
var ModelCmd = &cobra.Command{
Use: "model",
Expand Down Expand Up @@ -473,7 +600,14 @@ func init() {
listModelCmd.Flags().IntVarP(&pageNumberFlag, "page", "p", 1, "(optional) List next set of models with --page (default = 1)")
listModelCmd.Flags().BoolP("count", "c", false, "(optional) Get the number of models in total")
viewModelCmd.Flags().StringVarP(&outFormatFlag, "output-format", "o", "yaml", "(optional) format to display in [json|yaml]")
availableSubcommands = []*cobra.Command{listModelCmd, viewModelCmd, searchModelCmd}
pushModelCmd.Flags().StringVarP(&username, "username", "u", "", "Username for authentication")
pushModelCmd.Flags().StringVarP(&password, "password", "p", "", "Password for authentication")
pushModelCmd.Flags().StringVarP(&registry, "registry", "r", "", "Registry to push the model to")
pushModelCmd.Flags().StringVarP(&repository, "repository", "n", "", "Repository name to push the model to")
pushModelCmd.Flags().StringVarP(&tag, "tag", "t", "", "Tag for the model")
pullModelCmd.Flags().StringVarP(&registry, "registry", "r", "", "Registry to pull the model from")
pullModelCmd.Flags().StringVarP(&tag, "tag", "t", "", "Tag for the model")
availableSubcommands = []*cobra.Command{listModelCmd, viewModelCmd, searchModelCmd, pushModelCmd, pullModelCmd}
ModelCmd.AddCommand(availableSubcommands...)
}

Expand Down Expand Up @@ -530,4 +664,4 @@ func prettifyJson(model v1alpha1.Model) error {

// Any errors during the encoding process will be returned as an error.
return enc.Encode(model)
}
}