codeintel: Log errors in janitor but continue with best-effort (#11031)

cmd/precise-code-intel-bundle-manager/internal/janitor/bundleless_uploads.go

@@ -7,6 +7,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sourcegraph/sourcegraph/cmd/precise-code-intel-bundle-manager/internal/paths"
 	"github.com/sourcegraph/sourcegraph/internal/codeintel/gitserver"
+	"github.com/sourcegraph/sourcegraph/internal/vcs"
 )
 
 // removeProcessedUploadsWithoutBundleFile removes all processed upload records
@@ -32,7 +33,7 @@ func (j *Janitor) removeProcessedUploadsWithoutBundleFile() error {
 
 		deleted, err := j.db.DeleteUploadByID(ctx, id, func(repositoryID int) (string, error) {
 			tipCommit, err := gitserver.Head(ctx, j.db, repositoryID)
-			if err != nil {
+			if err != nil && !vcs.IsRepoNotExist(err) {
 				return "", errors.Wrap(err, "gitserver.Head")
 			}
 			return tipCommit, nil

cmd/precise-code-intel-bundle-manager/internal/janitor/free_space.go

@@ -21,12 +21,11 @@ func (j *Janitor) freeSpace() error {
 	diskSizeBytes := fs.Blocks * uint64(fs.Bsize)
 	freeBytes := fs.Bavail * uint64(fs.Bsize)
 	desiredFreeBytes := uint64(float64(diskSizeBytes) * float64(j.desiredPercentFree) / 100.0)
-
-	if freeBytes < desiredFreeBytes {
-		return j.evictBundles(uint64(desiredFreeBytes - freeBytes))
+	if freeBytes >= desiredFreeBytes {
+		return nil
 	}
 
-	return nil
+	return j.evictBundles(desiredFreeBytes - freeBytes)
 }
 
 // evictBundles removes completed upload recors from the database and then deletes the
@@ -77,7 +76,9 @@ func (j *Janitor) evictBundle() (uint64, bool, error) {
 	}
 
 	if err := os.Remove(path); err != nil {
-		return 0, false, err
+		j.metrics.Errors.Inc()
+		log15.Error("Failed to remove file", "path", path, "err", err)
+		return 0, true, nil
 	}
 
 	log15.Debug("Removed evicted bundle file", "id", id, "path", path)

cmd/precise-code-intel-bundle-manager/internal/janitor/old_database_part_files.go

@@ -29,8 +29,11 @@ func (j *Janitor) removeOldDatabasePartFiles() error {
 		}
 
 		path := filepath.Join(paths.DBsDir(j.bundleDir), fileInfo.Name())
+
 		if err := os.Remove(path); err != nil {
-			return err
+			j.metrics.Errors.Inc()
+			log15.Error("Failed to remove file", "path", path, "err", err)
+			continue
 		}
 
 		log15.Debug("Removed old database part file", "path", path, "age", age)

cmd/precise-code-intel-bundle-manager/internal/janitor/old_upload_files.go

@@ -28,8 +28,11 @@ func (j *Janitor) removeOldUploadFiles() error {
 		}
 
 		path := filepath.Join(paths.UploadsDir(j.bundleDir), fileInfo.Name())
+
 		if err := os.Remove(path); err != nil {
-			return err
+			j.metrics.Errors.Inc()
+			log15.Error("Failed to remove file", "path", path, "err", err)
+			continue
 		}
 
 		log15.Debug("Removed old upload file", "path", path, "age", age)

cmd/precise-code-intel-bundle-manager/internal/janitor/orphaned_bundle_files.go

@@ -45,7 +45,9 @@ func (j *Janitor) removeOrphanedBundleFiles() error {
 	for id, path := range pathsByID {
 		if state, exists := states[id]; !exists || state == "errored" {
 			if err := os.Remove(path); err != nil {
-				return err
+				j.metrics.Errors.Inc()
+				log15.Error("Failed to remove file", "path", path, "err", err)
+				continue
 			}
 
 			log15.Debug("Removed orphaned bundle file", "id", id, "path", path)
@@ -66,7 +68,7 @@ func (j *Janitor) databasePathsByID() (map[int]string, error) {
 	pathsByID := map[int]string{}
 	for _, fileInfo := range fileInfos {
 		if id, err := strconv.Atoi(strings.Split(fileInfo.Name(), ".")[0]); err == nil {
-			pathsByID[int(id)] = filepath.Join(paths.DBsDir(j.bundleDir), fileInfo.Name())
+			pathsByID[id] = filepath.Join(paths.DBsDir(j.bundleDir), fileInfo.Name())
 		}
 	}