Merge pull request #23 from sudo-bmitch/pr-blob-delete

Add blob delete API

blob.go

@@ -64,6 +64,40 @@ func (s *server) blobGet(repoStr, arg string) http.HandlerFunc {
 	}
 }
 
+func (s *server) blobDelete(repoStr, arg string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		repo, err := s.store.RepoGet(repoStr)
+		if err != nil {
+			if errors.Is(err, types.ErrRepoNotAllowed) {
+				w.WriteHeader(http.StatusBadRequest)
+				_ = types.ErrRespJSON(w, types.ErrInfoNameInvalid("repository name is not allowed"))
+				return
+			}
+			w.WriteHeader(http.StatusInternalServerError)
+			s.log.Info("failed to get repo", "err", err, "repo", repoStr, "arg", arg)
+			return
+		}
+		d, err := digest.Parse(arg)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			_ = types.ErrRespJSON(w, types.ErrInfoDigestInvalid("digest cannot be parsed"))
+			return
+		}
+		err = repo.BlobDelete(d)
+		if err != nil {
+			s.log.Debug("failed to delete blob", "err", err, "repo", repoStr, "digest", d.String())
+			if errors.Is(err, types.ErrNotFound) {
+				w.WriteHeader(http.StatusNotFound)
+				_ = types.ErrRespJSON(w, types.ErrInfoBlobUnknown("blob was not found"))
+			} else {
+				w.WriteHeader(http.StatusInternalServerError)
+			}
+			return
+		}
+		w.WriteHeader(http.StatusAccepted)
+	}
+}
+
 // TODO: replace sessions with a cache that expires entries (with cancel) after some time and automatically cancels on shutdown.
 // TODO: cache can also limit concurrent sessions.
 var (

cmd/olareg/serve.go

@@ -13,11 +13,13 @@ import (
 )
 
 type serveOpts struct {
-	root      *rootOpts
-	addr      string
-	port      int
-	storeType string
-	storeDir  string
+	root        *rootOpts
+	addr        string
+	port        int
+	storeType   string
+	storeDir    string
+	apiBlobDel  bool
+	apiReferrer bool
 }
 
 func newServeCmd(root *rootOpts) *cobra.Command {
@@ -34,6 +36,8 @@ func newServeCmd(root *rootOpts) *cobra.Command {
 	newCmd.Flags().IntVar(&opts.port, "port", 80, "listener port")
 	newCmd.Flags().StringVar(&opts.storeDir, "dir", ".", "root directory for storage")
 	newCmd.Flags().StringVar(&opts.storeType, "store-type", "dir", "storage type (dir, mem)")
+	newCmd.Flags().BoolVar(&opts.apiBlobDel, "api-blob-delete", false, "enable blob delete API")
+	newCmd.Flags().BoolVar(&opts.apiReferrer, "api-referrer", true, "enable referrer API")
 	return newCmd
 }
 
@@ -47,6 +51,10 @@ func (opts *serveOpts) run(cmd *cobra.Command, args []string) error {
 		StoreType: storeType,
 		RootDir:   opts.storeDir,
 		Log:       opts.root.log,
+		API: config.ConfigAPI{
+			BlobDelete: config.ConfigAPIBlobDelete{Enabled: &opts.apiBlobDel},
+			Referrer:   config.ConfigAPIReferrer{Enabled: &opts.apiReferrer},
+		},
 	}
 	handler := olareg.New(conf)
 	listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", opts.addr, opts.port))

config/config.go

@@ -17,11 +17,11 @@ const (
 )
 
 type Config struct {
-	StoreType        Store
-	RootDir          string
-	Log              slog.Logger
-	ManifestLimit    int64
-	DisableReferrers bool
+	StoreType     Store
+	RootDir       string
+	Log           slog.Logger
+	ManifestLimit int64
+	API           ConfigAPI
 	// TODO: TLS and listener options? not needed here if only providing handler
 	// TODO: GC policy, delete untagged? timeouts for partial blobs?
 	// TODO: proxy settings, pull only, or push+pull cache
@@ -30,6 +30,19 @@ type Config struct {
 	// TODO: allowed actions: get/head, put, delete, catalog
 }
 
+type ConfigAPI struct {
+	BlobDelete ConfigAPIBlobDelete
+	Referrer   ConfigAPIReferrer
+}
+
+type ConfigAPIBlobDelete struct {
+	Enabled *bool
+}
+
+type ConfigAPIReferrer struct {
+	Enabled *bool
+}
+
 func (s Store) MarshalText() ([]byte, error) {
 	var ret string
 	switch s {

internal/store/dir.go

@@ -30,23 +30,23 @@ const (
 )
 
 type dir struct {
-	mu               sync.Mutex
-	root             string
-	repos            map[string]*dirRepo // TODO: switch to storing these in a cache that expires from memory
-	log              slog.Logger
-	disableReferrers bool
+	mu    sync.Mutex
+	root  string
+	repos map[string]*dirRepo // TODO: switch to storing these in a cache that expires from memory
+	log   slog.Logger
+	conf  config.Config
 }
 
 type dirRepo struct {
-	timeCheck        time.Time
-	timeMod          time.Time
-	mu               sync.Mutex
-	name             string
-	path             string
-	exists           bool
-	index            types.Index
-	log              slog.Logger
-	disableReferrers bool
+	timeCheck time.Time
+	timeMod   time.Time
+	mu        sync.Mutex
+	name      string
+	path      string
+	exists    bool
+	index     types.Index
+	log       slog.Logger
+	conf      config.Config
 }
 
 type dirRepoUpload struct {
@@ -69,6 +69,7 @@ func NewDir(conf config.Config, opts ...Opts) Store {
 		root:  conf.RootDir,
 		repos: map[string]*dirRepo{},
 		log:   sc.log,
+		conf:  conf,
 	}
 	if d.log == nil {
 		d.log = slog.Null{}
@@ -88,10 +89,10 @@ func (d *dir) RepoGet(repoStr string) (Repo, error) {
 		return nil, fmt.Errorf("repo %s cannot contain %s, %s, or %s%.0w", repoStr, indexFile, layoutFile, blobsDir, types.ErrRepoNotAllowed)
 	}
 	dr := dirRepo{
-		path:             filepath.Join(d.root, repoStr),
-		name:             repoStr,
-		log:              d.log,
-		disableReferrers: d.disableReferrers,
+		path: filepath.Join(d.root, repoStr),
+		name: repoStr,
+		log:  d.log,
+		conf: d.conf,
 	}
 	d.repos[repoStr] = &dr
 	statDir, err := os.Stat(dr.path)
@@ -186,6 +187,26 @@ func (dr *dirRepo) BlobCreate(opts ...BlobOpt) (BlobCreator, error) {
 	}, nil
 }
 
+// BlobDelete deletes an entry from the CAS.
+func (dr *dirRepo) BlobDelete(d digest.Digest) error {
+	if !dr.exists {
+		return fmt.Errorf("repo does not exist %s: %w", dr.name, types.ErrNotFound)
+	}
+	filename := filepath.Join(dr.path, blobsDir, d.Algorithm().String(), d.Encoded())
+	fi, err := os.Stat(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("failed to stat %s: %w", d.String(), types.ErrNotFound)
+		}
+		return fmt.Errorf("failed to stat %s: %w", d.String(), err)
+	}
+	if fi.IsDir() {
+		return fmt.Errorf("invalid blob %s: %s is a directory", d.String(), filename)
+	}
+	err = os.Remove(filename)
+	return err
+}
+
 func (dr *dirRepo) repoInit() error {
 	dr.mu.Lock()
 	defer dr.mu.Unlock()
@@ -277,7 +298,7 @@ func (dr *dirRepo) repoLoad(force, locked bool) error {
 		}
 	}
 	dr.timeMod = stat.ModTime()
-	if !dr.disableReferrers {
+	if boolDefault(dr.conf.API.Referrer.Enabled, true) {
 		mod, err := referrerConvert(dr, &dr.index)
 		if err != nil {
 			return err

internal/store/mem.go

@@ -14,18 +14,18 @@ import (
 )
 
 type mem struct {
-	mu               sync.Mutex
-	repos            map[string]*memRepo
-	disableReferrers bool
-	log              slog.Logger
+	mu    sync.Mutex
+	repos map[string]*memRepo
+	log   slog.Logger
+	conf  config.Config
 }
 
 type memRepo struct {
-	mu               sync.Mutex
-	index            types.Index
-	blobs            map[digest.Digest][]byte
-	disableReferrers bool
-	log              slog.Logger
+	mu    sync.Mutex
+	index types.Index
+	blobs map[digest.Digest][]byte
+	log   slog.Logger
+	conf  config.Config
 }
 
 type memRepoUpload struct {
@@ -42,9 +42,9 @@ func NewMem(conf config.Config, opts ...Opts) Store {
 		opt(&sc)
 	}
 	m := &mem{
-		repos:            map[string]*memRepo{},
-		disableReferrers: conf.DisableReferrers,
-		log:              sc.log,
+		repos: map[string]*memRepo{},
+		log:   sc.log,
+		conf:  conf,
 	}
 	if m.log == nil {
 		m.log = slog.Null{}
@@ -62,9 +62,9 @@ func (m *mem) RepoGet(repoStr string) (Repo, error) {
 		index: types.Index{
 			Manifests: []types.Descriptor{},
 		},
-		blobs:            map[digest.Digest][]byte{},
-		disableReferrers: m.disableReferrers,
-		log:              m.log,
+		blobs: map[digest.Digest][]byte{},
+		log:   m.log,
+		conf:  m.conf,
 	}
 	m.repos[repoStr] = mr
 	return mr, nil
@@ -143,6 +143,18 @@ func (mr *memRepo) BlobCreate(opts ...BlobOpt) (BlobCreator, error) {
 	}, nil
 }
 
+// BlobDelete deletes an entry from the CAS.
+func (mr *memRepo) BlobDelete(d digest.Digest) error {
+	mr.mu.Lock()
+	_, ok := mr.blobs[d]
+	delete(mr.blobs, d)
+	mr.mu.Unlock()
+	if !ok {
+		return types.ErrNotFound
+	}
+	return nil
+}
+
 // Write sends data to the buffer.
 func (mru *memRepoUpload) Write(p []byte) (int, error) {
 	return mru.w.Write(p)

internal/store/store.go

@@ -32,6 +32,8 @@ type Repo interface {
 	BlobGet(d digest.Digest) (io.ReadSeekCloser, error)
 	// BlobCreate is used to create a new blob.
 	BlobCreate(opts ...BlobOpt) (BlobCreator, error)
+	// BlobDelete deletes an entry from the CAS.
+	BlobDelete(d digest.Digest) error
 }
 
 type BlobOpt func(*blobConfig)
@@ -265,3 +267,10 @@ func repoGetIndex(repo Repo, d types.Descriptor) (types.Index, error) {
 	}
 	return i, nil
 }
+
+func boolDefault(b *bool, def bool) bool {
+	if b != nil {
+		return *b
+	}
+	return def
+}

manifest.go

@@ -49,7 +49,7 @@ func (s *server) manifestDelete(repoStr, arg string) http.HandlerFunc {
 			return
 		}
 		// if referrers is enabled, get the manifest to check for a subject
-		if !s.conf.DisableReferrers {
+		if boolDefault(s.conf.API.Referrer.Enabled, true) {
 			// wrap in a func to allow a return from errors without breaking the actual delete
 			err = func() error {
 				rdr, err := repo.BlobGet(desc.Digest)
@@ -63,6 +63,9 @@ func (s *server) manifestDelete(repoStr, arg string) http.HandlerFunc {
 				}
 				subject, refDesc, err := types.ManifestReferrerDescriptor(raw, desc)
 				if err != nil {
+					if errors.Is(types.ErrNotFound, err) {
+						return nil
+					}
 					return err
 				}
 				err = s.referrerDelete(repo, subject.Digest, refDesc)
@@ -269,7 +272,7 @@ func (s *server) manifestPut(repoStr, arg string) http.HandlerFunc {
 				s.log.Debug("manifest blobs missing", "repo", repoStr, "arg", arg, "mediaType", mt, "errList", eList)
 				return
 			}
-			if m.Subject != nil && m.Subject.Digest != "" && !s.conf.DisableReferrers {
+			if m.Subject != nil && m.Subject.Digest != "" && boolDefault(s.conf.API.Referrer.Enabled, true) {
 				subject = m.Subject.Digest
 				referrer = &types.Descriptor{
 					MediaType:    mt,
@@ -300,7 +303,7 @@ func (s *server) manifestPut(repoStr, arg string) http.HandlerFunc {
 				s.log.Debug("child manifests missing", "repo", repoStr, "arg", arg, "mediaType", mt, "errList", eList)
 				return
 			}
-			if m.Subject != nil && m.Subject.Digest != "" && !s.conf.DisableReferrers {
+			if m.Subject != nil && m.Subject.Digest != "" && boolDefault(s.conf.API.Referrer.Enabled, true) {
 				subject = m.Subject.Digest
 				referrer = &types.Descriptor{
 					MediaType:    mt,

olareg.go

@@ -81,6 +81,10 @@ func (s *server) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 			// handle blob get
 			s.blobGet(matches[0], matches[1]).ServeHTTP(resp, req)
 			return
+		} else if req.Method == http.MethodDelete && boolDefault(s.conf.API.BlobDelete.Enabled, false) {
+			// handle blob delete
+			s.blobDelete(matches[0], matches[1]).ServeHTTP(resp, req)
+			return
 		} else if matches[1] == "uploads" && req.Method == http.MethodPost {
 			// handle blob post
 			s.blobUploadPost(matches[0]).ServeHTTP(resp, req)
@@ -90,7 +94,7 @@ func (s *server) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
 			resp.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
-	} else if matches, ok := matchV2(pathEl, "...", "referrers", "*"); ok && !s.conf.DisableReferrers {
+	} else if matches, ok := matchV2(pathEl, "...", "referrers", "*"); ok && boolDefault(s.conf.API.Referrer.Enabled, true) {
 		if req.Method == http.MethodGet || req.Method == http.MethodHead {
 			// handle referrer get
 			s.referrerGet(matches[0], matches[1]).ServeHTTP(resp, req)
@@ -177,3 +181,10 @@ func (s *server) v2Ping(resp http.ResponseWriter, req *http.Request) {
 		_, _ = resp.Write([]byte("{}"))
 	}
 }
+
+func boolDefault(b *bool, def bool) bool {
+	if b != nil {
+		return *b
+	}
+	return def
+}