Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions cmd/cli/readline/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,3 @@ import (
)

var ErrInterrupt = errors.New("Interrupt")

type InterruptError struct {
Line []rune
}

func (*InterruptError) Error() string {
return "Interrupted"
}
4 changes: 0 additions & 4 deletions cmd/cli/readline/history.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,6 @@ func (h *History) Compact() {
}
}

func (h *History) Clear() {
h.Buf.Clear()
}

func (h *History) Prev() (line string) {
if h.Pos > 0 {
h.Pos--
Expand Down
4 changes: 0 additions & 4 deletions cmd/cli/readline/readline.go
Original file line number Diff line number Diff line change
Expand Up @@ -241,10 +241,6 @@ func (i *Instance) Readline() (string, error) {
}
}

func (i *Instance) HistoryEnable() {
i.History.Enabled = true
}

func (i *Instance) HistoryDisable() {
i.History.Enabled = false
}
Expand Down
27 changes: 0 additions & 27 deletions pkg/distribution/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -455,33 +455,6 @@ func (b *Builder) WithChatTemplateFile(path string) (*Builder, error) {
}, nil
}

// WithConfigArchive adds a config archive (tar) file to the artifact.
func (b *Builder) WithConfigArchive(path string) (*Builder, error) {
// Check if config archive already exists.
layers, err := b.model.Layers()
if err != nil {
return nil, fmt.Errorf("get model layers: %w", err)
}

for _, layer := range layers {
mediaType, mediaTypeErr := layer.MediaType()
if mediaTypeErr == nil && mediaType == types.MediaTypeVLLMConfigArchive {
return nil, fmt.Errorf("model already has a config archive layer")
}
}

mt := b.resolveLayerMediaType(types.MediaTypeVLLMConfigArchive)
configLayer, err := partial.NewLayer(path, mt)
if err != nil {
return nil, fmt.Errorf("config archive layer from %q: %w", path, err)
}
return &Builder{
model: mutate.AppendLayers(b.model, configLayer),
originalLayers: b.originalLayers,
outputFormat: b.outputFormat,
}, nil
}

// Target represents a build target
type Target interface {
Write(context.Context, types.ModelArtifact, io.Writer) error
Expand Down
19 changes: 0 additions & 19 deletions pkg/distribution/builder/from_directory.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

"github.com/docker/model-runner/pkg/distribution/files"
"github.com/docker/model-runner/pkg/distribution/format"
"github.com/docker/model-runner/pkg/distribution/internal/mutate"
"github.com/docker/model-runner/pkg/distribution/internal/partial"
"github.com/docker/model-runner/pkg/distribution/modelpack"
"github.com/docker/model-runner/pkg/distribution/oci"
Expand Down Expand Up @@ -382,21 +381,3 @@ func fileTypeToMediaType(ft files.FileType) oci.MediaType {
return types.MediaTypeModelFile
}
}

// WithFileLayer adds an individual file layer with a relative path annotation.
// This is useful for adding files that should be extracted to a specific path.
func (b *Builder) WithFileLayer(absPath, relPath string) (*Builder, error) {
// Classify the file to determine media type
fileType := files.Classify(absPath)
mediaType := fileTypeToMediaType(fileType)

layer, err := partial.NewLayerWithRelativePath(absPath, relPath, mediaType)
if err != nil {
return nil, fmt.Errorf("file layer from %q: %w", absPath, err)
}

return &Builder{
model: mutate.AppendLayers(b.model, layer),
originalLayers: b.originalLayers,
}, nil
}
9 changes: 0 additions & 9 deletions pkg/distribution/huggingface/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,6 @@ func WithToken(token string) ClientOption {
}
}

// WithTransport sets the HTTP transport for the client
func WithTransport(transport http.RoundTripper) ClientOption {
return func(c *Client) {
if transport != nil {
c.httpClient.Transport = transport
}
}
}

// WithUserAgent sets the User-Agent header for requests
func WithUserAgent(userAgent string) ClientOption {
return func(c *Client) {
Expand Down
5 changes: 0 additions & 5 deletions pkg/distribution/huggingface/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,3 @@ func (pr *progressReader) Read(p []byte) (n int, err error) {
}
return n, err
}

// DownloadSingleFile downloads a single file and returns its local path
func (d *Downloader) DownloadSingleFile(ctx context.Context, file RepoFile) (string, error) {
return d.downloadFileWithProgress(ctx, file, uint64(file.ActualSize()), nil)
}
11 changes: 0 additions & 11 deletions pkg/distribution/internal/mutate/mutate.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,3 @@ func ContextSize(mdl types.ModelArtifact, cs int32) types.ModelArtifact {
contextSize: &cs,
}
}

// SetManifestOptions applies manifest-level metadata (config media type and
// artifact type) to the given model. The options propagate through subsequent
// mutate wrappers via GetManifestOptions.
func SetManifestOptions(mdl types.ModelArtifact, configMT oci.MediaType, artifactType string) types.ModelArtifact {
return &model{
base: mdl,
configMediaType: configMT,
artifactType: artifactType,
}
}
9 changes: 0 additions & 9 deletions pkg/distribution/internal/progress/reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,15 +142,6 @@ func WriteError(w io.Writer, message string, mode oci.Mode) error {
})
}

// WriteWarning writes a warning message
func WriteWarning(w io.Writer, message string, mode oci.Mode) error {
return write(w, oci.ProgressMessage{
Type: oci.TypeWarning,
Message: message,
Mode: mode,
})
}

// write writes a JSON-formatted progress message to the writer
func write(w io.Writer, msg oci.ProgressMessage) error {
if w == nil {
Expand Down
21 changes: 0 additions & 21 deletions pkg/distribution/internal/utils/utils.go

This file was deleted.

8 changes: 0 additions & 8 deletions pkg/distribution/oci/remote/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ type options struct {
userAgent string
auth authn.Authenticator
keychain authn.Keychain
progress chan<- oci.Update
plainHTTP bool
}

Expand Down Expand Up @@ -85,13 +84,6 @@ func WithAuthFromKeychain(kc authn.Keychain) Option {
}
}

// WithProgress sets a channel for receiving progress updates.
func WithProgress(ch chan<- oci.Update) Option {
return func(o *options) {
o.progress = ch
}
}

// WithPlainHTTP allows connecting to registries using plain HTTP instead of HTTPS.
func WithPlainHTTP(plain bool) Option {
return func(o *options) {
Expand Down
11 changes: 0 additions & 11 deletions pkg/distribution/registry/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,6 @@ func WithUserAgent(userAgent string) ClientOption {
}
}

func WithAuthConfig(username, password string) ClientOption {
return func(c *Client) {
if username != "" && password != "" {
c.auth = &authn.Basic{
Username: username,
Password: password,
}
}
}
}

// WithAuth sets a custom authenticator.
func WithAuth(auth authn.Authenticator) ClientOption {
return func(c *Client) {
Expand Down
36 changes: 0 additions & 36 deletions pkg/distribution/tarball/file.go

This file was deleted.

13 changes: 0 additions & 13 deletions pkg/distribution/tarball/reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,6 @@ type Reader struct {
done bool
}

type Blob struct {
diffID oci.Hash
rc io.ReadCloser
}

func (b Blob) DiffID() (oci.Hash, error) {
return b.diffID, nil
}

func (b Blob) Uncompressed() (io.ReadCloser, error) {
return b.rc, nil
}

func (r *Reader) Next() (oci.Hash, error) {
for {
hdr, err := r.tr.Next()
Expand Down
7 changes: 0 additions & 7 deletions pkg/envconfig/envconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,6 @@ func Var(key string) string {
return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")
}

// String returns a lazy string accessor for the given environment variable.
func String(key string) func() string {
return func() string {
return Var(key)
}
}

// BoolWithDefault returns a lazy bool accessor for the given environment variable,
// allowing a caller-specified default. If the variable is set but cannot be parsed
// as a bool, the defaultValue is returned.
Expand Down
15 changes: 0 additions & 15 deletions pkg/inference/models/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,6 @@ type ModelPushRequest struct {
BearerToken string `json:"bearer-token,omitempty"`
}

// SimpleModel is a wrapper that allows creating a model with modified configuration
type SimpleModel struct {
types.Model
ConfigValue types.ModelConfig
DescriptorValue types.Descriptor
}

func (s *SimpleModel) Config() (types.ModelConfig, error) {
return s.ConfigValue, nil
}

func (s *SimpleModel) Descriptor() (types.Descriptor, error) {
return s.DescriptorValue, nil
}

// ToOpenAIList converts the model list to its OpenAI API representation. This function never
// returns a nil slice (though it may return an empty slice).
func ToOpenAIList(l []types.Model) (*OpenAIModelList, error) {
Expand Down
6 changes: 0 additions & 6 deletions pkg/inference/scheduling/http_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -555,12 +555,6 @@ func (h *HTTPHandler) RebuildRoutes(allowedOrigins []string) {
h.httpHandler = middleware.CorsMiddleware(allowedOrigins, h.router)
}

// GetLlamaCppSocket delegates to the scheduler's business logic.
// Required by metrics.SchedulerInterface.
func (h *HTTPHandler) GetLlamaCppSocket() (string, error) {
return h.scheduler.GetLlamaCppSocket()
}

// GetAllActiveRunners delegates to the scheduler's business logic.
// Required by metrics.SchedulerInterface.
func (h *HTTPHandler) GetAllActiveRunners() []metrics.ActiveRunner {
Expand Down
32 changes: 0 additions & 32 deletions pkg/inference/scheduling/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package scheduling

import (
"context"
"errors"
"fmt"
"net/http"
"slices"
Expand All @@ -11,7 +10,6 @@ import (
"github.com/docker/model-runner/pkg/distribution/types"
"github.com/docker/model-runner/pkg/inference"
"github.com/docker/model-runner/pkg/inference/backends/diffusers"
"github.com/docker/model-runner/pkg/inference/backends/llamacpp"
"github.com/docker/model-runner/pkg/inference/backends/mlx"
"github.com/docker/model-runner/pkg/inference/backends/sglang"
"github.com/docker/model-runner/pkg/inference/backends/vllm"
Expand Down Expand Up @@ -317,36 +315,6 @@ func (s *Scheduler) GetAllActiveRunners() []metrics.ActiveRunner {
return activeRunners
}

// GetLlamaCppSocket returns the Unix socket path for an active llama.cpp runner
func (s *Scheduler) GetLlamaCppSocket() (string, error) {
runningBackends := s.getLoaderStatus(context.Background())

if !s.loader.lock(context.Background()) {
return "", errors.New("failed to acquire loader lock")
}
defer s.loader.unlock()

// Look for an active llama.cpp backend
for _, backend := range runningBackends {
if backend.BackendName == llamacpp.Name {
mode, ok := inference.ParseBackendMode(backend.Mode)
if !ok {
s.log.Warn("Unknown backend mode, defaulting to completion", "mode", backend.Mode)
}
// Find the runner slot for this backend/model combination
// We iterate through all runners since we don't know the draftModelID
for key, runnerInfo := range s.loader.runners {
if key.backend == backend.BackendName && key.modelID == backend.ModelName && key.mode == mode {
// Use the RunnerSocketPath function to get the socket path
return RunnerSocketPath(runnerInfo.slot)
}
}
}
}

return "", errors.New("no active llama.cpp backend found")
}

// ConfigureRunner configures a runner for a specific model and backend.
// It handles all the business logic of configuration including parsing flags,
// determining mode, selecting backend, and setting runner configuration.
Expand Down
14 changes: 14 additions & 0 deletions pkg/metrics/aggregated_handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,20 @@ import (
"github.com/prometheus/common/model"
)

// SchedulerInterface defines the methods we need from the scheduler
type SchedulerInterface interface {
GetRunningBackends(w http.ResponseWriter, r *http.Request)
GetAllActiveRunners() []ActiveRunner
}

// ActiveRunner contains information about an active runner
type ActiveRunner struct {
BackendName string
ModelName string
Mode string
Socket string
}

// AggregatedMetricsHandler collects metrics from all active runners and aggregates them with labels
type AggregatedMetricsHandler struct {
log logging.Logger
Expand Down
Loading
Loading