diff --git a/internal/config/config.go b/internal/config/config.go index ad0acc0..2984d5c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -391,6 +391,19 @@ func (c *Config) Validate() error { const defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default +// ParseMaxSize returns the maximum cache size in bytes. +// Returns 0 if unset or explicitly disabled (meaning unlimited). +func (c *Config) ParseMaxSize() int64 { + if c.Storage.MaxSize == "" || c.Storage.MaxSize == "0" { + return 0 + } + size, err := ParseSize(c.Storage.MaxSize) + if err != nil { + return 0 + } + return size +} + // ParseMetadataTTL returns the metadata TTL duration. // Returns 5 minutes if unset, 0 if explicitly disabled. func (c *Config) ParseMetadataTTL() time.Duration { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6e8c3a0..25f3488 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -303,6 +303,31 @@ func TestLoadFileNotFound(t *testing.T) { } } +func TestParseMaxSize(t *testing.T) { + tests := []struct { + name string + maxSize string + want int64 + }{ + {"empty means unlimited", "", 0}, + {"zero means unlimited", "0", 0}, + {"10GB", "10GB", 10 * 1024 * 1024 * 1024}, + {"500MB", "500MB", 500 * 1024 * 1024}, + {"invalid returns 0", "invalid", 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.MaxSize = tt.maxSize + got := cfg.ParseMaxSize() + if got != tt.want { + t.Errorf("ParseMaxSize() = %d, want %d", got, tt.want) + } + }) + } +} + func TestParseMetadataTTL(t *testing.T) { tests := []struct { name string diff --git a/internal/server/eviction.go b/internal/server/eviction.go new file mode 100644 index 0000000..4173bd5 --- /dev/null +++ b/internal/server/eviction.go @@ -0,0 +1,105 @@ +package server + +import ( + "context" + "log/slog" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +const ( + evictionInterval = 1 * time.Minute + evictionBatch = 50 +) + +func (s *Server) startEvictionLoop(ctx context.Context) { + maxSize := s.cfg.ParseMaxSize() + if maxSize <= 0 { + return + } + + s.logger.Info("cache eviction enabled", "max_size", s.cfg.Storage.MaxSize) + + ticker := time.NewTicker(evictionInterval) + defer ticker.Stop() + + s.runEviction(ctx, maxSize) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.runEviction(ctx, maxSize) + } + } +} + +func (s *Server) runEviction(ctx context.Context, maxSize int64) { + evictLRU(ctx, s.db, s.storage, s.logger, maxSize) +} + +func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, maxSize int64) { + totalSize, err := db.GetTotalCacheSize() + if err != nil { + logger.Warn("eviction: failed to get cache size", "error", err) + return + } + + if totalSize <= maxSize { + return + } + + logger.Info("eviction: cache size exceeds limit, evicting", + "current_size", totalSize, "max_size", maxSize) + + evicted := 0 + freedBytes := int64(0) + + for totalSize-freedBytes > maxSize { + artifacts, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + if err != nil { + logger.Warn("eviction: failed to get LRU artifacts", "error", err) + return + } + if len(artifacts) == 0 { + break + } + + for _, art := range artifacts { + if totalSize-freedBytes <= maxSize { + break + } + + if !art.StoragePath.Valid { + continue + } + + if err := store.Delete(ctx, art.StoragePath.String); err != nil { + logger.Warn("eviction: failed to delete from storage", + "path", art.StoragePath.String, "error", err) + continue + } + + if err := db.ClearArtifactCache(art.VersionPURL, art.Filename); err != nil { + logger.Warn("eviction: failed to clear artifact record", + "version_purl", art.VersionPURL, "filename", art.Filename, "error", err) + continue + } + + size := int64(0) + if art.Size.Valid { + size = art.Size.Int64 + } + freedBytes += size + evicted++ + } + } + + if evicted > 0 { + logger.Info("eviction: completed", + "evicted", evicted, "freed_bytes", freedBytes) + } +} diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go new file mode 100644 index 0000000..9fa9e6b --- /dev/null +++ b/internal/server/eviction_test.go @@ -0,0 +1,290 @@ +package server + +import ( + "context" + "database/sql" + "io" + "log/slog" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +func setupEvictionTest(t *testing.T) (*database.DB, *storage.Filesystem) { + t.Helper() + + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + storagePath := filepath.Join(tempDir, "artifacts") + + db, err := database.Create(dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + + store, err := storage.NewFilesystem(storagePath) + if err != nil { + _ = db.Close() + t.Fatalf("failed to create storage: %v", err) + } + + t.Cleanup(func() { + _ = db.Close() + }) + + return db, store +} + +func seedArtifact(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, name string, dataSize int, accessedAt time.Time) { + t.Helper() + + pkgPURL := "pkg:npm/" + name + versionPURL := pkgPURL + "@1.0.0" + filename := name + "-1.0.0.tgz" + + if err := db.UpsertPackage(&database.Package{ + PURL: pkgPURL, + Ecosystem: "npm", + Name: name, + }); err != nil { + t.Fatalf("failed to upsert package: %v", err) + } + + if err := db.UpsertVersion(&database.Version{ + PURL: versionPURL, + PackagePURL: pkgPURL, + }); err != nil { + t.Fatalf("failed to upsert version: %v", err) + } + + storagePath := storage.ArtifactPath("npm", "", name, "1.0.0", filename) + data := strings.NewReader(strings.Repeat("x", dataSize)) + size, hash, err := store.Store(ctx, storagePath, data) + if err != nil { + t.Fatalf("failed to store artifact: %v", err) + } + + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.com/" + filename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + ContentType: sql.NullString{String: "application/gzip", Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + LastAccessedAt: sql.NullTime{Time: accessedAt, Valid: true}, + }); err != nil { + t.Fatalf("failed to upsert artifact: %v", err) + } +} + +func TestEvictLRU_NoEvictionWhenUnderLimit(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + seedArtifact(t, ctx, db, store, "pkg-a", 100, time.Now()) + + evictLRU(ctx, db, store, logger, 1024) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 1 { + t.Errorf("expected 1 cached artifact, got %d", count) + } +} + +func TestEvictLRU_EvictsOldestFirst(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + now := time.Now() + seedArtifact(t, ctx, db, store, "old-pkg", 500, now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "mid-pkg", 500, now.Add(-1*time.Hour)) + seedArtifact(t, ctx, db, store, "new-pkg", 500, now) + + // Total is 1500 bytes, limit to 1100 so only the oldest gets evicted + evictLRU(ctx, db, store, logger, 1100) + + // old-pkg should be evicted + art, err := db.GetArtifact("pkg:npm/old-pkg@1.0.0", "old-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if art.StoragePath.Valid { + t.Error("expected old-pkg to be evicted (storage_path should be NULL)") + } + + // mid-pkg and new-pkg should remain + art, err = db.GetArtifact("pkg:npm/mid-pkg@1.0.0", "mid-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if !art.StoragePath.Valid { + t.Error("expected mid-pkg to remain cached") + } + + art, err = db.GetArtifact("pkg:npm/new-pkg@1.0.0", "new-pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if !art.StoragePath.Valid { + t.Error("expected new-pkg to remain cached") + } + + // Storage file should be removed for old-pkg + storagePath := storage.ArtifactPath("npm", "", "old-pkg", "1.0.0", "old-pkg-1.0.0.tgz") + exists, _ := store.Exists(ctx, storagePath) + if exists { + t.Error("expected old-pkg file to be deleted from storage") + } +} + +func TestEvictLRU_EvictsMultipleToGetUnderLimit(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + now := time.Now() + seedArtifact(t, ctx, db, store, "pkg-1", 400, now.Add(-4*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-2", 400, now.Add(-3*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-3", 400, now.Add(-2*time.Hour)) + seedArtifact(t, ctx, db, store, "pkg-4", 400, now) + + // Total is 1600 bytes, limit to 900 so pkg-1 and pkg-2 get evicted + evictLRU(ctx, db, store, logger, 900) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 2 { + t.Errorf("expected 2 cached artifacts remaining, got %d", count) + } + + // Verify the right ones remain + for _, name := range []string{"pkg-3", "pkg-4"} { + art, err := db.GetArtifact("pkg:npm/"+name+"@1.0.0", name+"-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact %s: %v", name, err) + } + if !art.StoragePath.Valid { + t.Errorf("expected %s to remain cached", name) + } + } +} + +func TestEvictLRU_NothingToEvictWhenEmpty(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + // Should not panic or error with no artifacts + evictLRU(ctx, db, store, logger, 1024) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 0 { + t.Errorf("expected 0 cached artifacts, got %d", count) + } +} + +func TestEvictLRU_StorageFileDeleted(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + seedArtifact(t, ctx, db, store, "delete-me", 1000, time.Now().Add(-1*time.Hour)) + + storagePath := storage.ArtifactPath("npm", "", "delete-me", "1.0.0", "delete-me-1.0.0.tgz") + exists, _ := store.Exists(ctx, storagePath) + if !exists { + t.Fatal("expected artifact file to exist before eviction") + } + + evictLRU(ctx, db, store, logger, 500) + + exists, _ = store.Exists(ctx, storagePath) + if exists { + t.Error("expected artifact file to be deleted after eviction") + } + + art, err := db.GetArtifact("pkg:npm/delete-me@1.0.0", "delete-me-1.0.0.tgz") + if err != nil { + t.Fatalf("failed to get artifact: %v", err) + } + if art.StoragePath.Valid { + t.Error("expected storage_path to be NULL after eviction") + } + if art.Size.Valid { + t.Error("expected size to be NULL after eviction") + } +} + +func TestStartEvictionLoop_UnlimitedSkips(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test.db") + storagePath := filepath.Join(tempDir, "artifacts") + + db, err := database.Create(dbPath) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + store, err := storage.NewFilesystem(storagePath) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + cfg := defaultTestConfig(storagePath, dbPath) + + s := &Server{ + cfg: cfg, + db: db, + storage: store, + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + // Should return immediately since max_size is empty (unlimited) + done := make(chan struct{}) + go func() { + s.startEvictionLoop(ctx) + close(done) + }() + + select { + case <-done: + // Good, returned immediately + case <-time.After(1 * time.Second): + t.Error("startEvictionLoop should return immediately when max_size is unlimited") + cancel() + } +} + +func defaultTestConfig(storagePath, dbPath string) *config.Config { + return &config.Config{ + Listen: ":8080", + BaseURL: "http://localhost:8080", + Storage: config.StorageConfig{Path: storagePath, MaxSize: ""}, + Database: config.DatabaseConfig{ + Driver: "sqlite", + Path: dbPath, + }, + Log: config.LogConfig{Level: "info", Format: "text"}, + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5d544a2..2ae0e69 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -261,6 +261,7 @@ func (s *Server) Start() error { "storage", s.storage.URL(), "database", s.cfg.Database.Path) go s.updateCacheStatsMetrics() + go s.startEvictionLoop(bgCtx) return s.http.ListenAndServe() }