Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
version: "2"

linters:
enable:
- gocritic
- gocognit
- gocyclo
- maintidx
- dupl
- mnd
- unparam
- ireturn
- goconst
- errcheck
settings:
goconst:
min-len: 4
min-occurrences: 5
ignore-tests: true
ignore-string-values:
- "^[a-z]+$"
exclusions:
rules:
- path: _test\.go
linters:
- goconst
- dupl
- mnd
54 changes: 31 additions & 23 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,10 +294,10 @@ func Default() *Config {
Gradle: GradleConfig{
BuildCache: GradleBuildCacheConfig{
ReadOnly: false,
MaxUploadSize: "100MB",
MaxUploadSize: defaultGradleMaxUploadSizeStr,
MaxAge: "168h",
MaxSize: "",
SweepInterval: "10m",
SweepInterval: defaultGradleSweepIntervalStr,
},
},
}
Expand Down Expand Up @@ -481,51 +481,59 @@ func (c *Config) Validate() error {
}
}

// Validate Gradle build cache upload size (always required and must be > 0).
if c.Gradle.BuildCache.MaxUploadSize == "" {
c.Gradle.BuildCache.MaxUploadSize = "100MB"
if err := c.Gradle.BuildCache.Validate(); err != nil {
return err
}

return nil
}

// Validate checks Gradle build cache settings, applying the default upload
// size if unset.
func (g *GradleBuildCacheConfig) Validate() error {
if g.MaxUploadSize == "" {
g.MaxUploadSize = defaultGradleMaxUploadSizeStr
}
uploadSize, err := ParseSize(c.Gradle.BuildCache.MaxUploadSize)
uploadSize, err := ParseSize(g.MaxUploadSize)
if err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_upload_size: %w", err)
}
if uploadSize <= 0 {
return fmt.Errorf("invalid gradle.build_cache.max_upload_size %q: must be > 0", c.Gradle.BuildCache.MaxUploadSize)
return fmt.Errorf("invalid gradle.build_cache.max_upload_size %q: must be > 0", g.MaxUploadSize)
}

// Validate Gradle max age if specified.
if c.Gradle.BuildCache.MaxAge != "" && c.Gradle.BuildCache.MaxAge != "0" {
if _, err := time.ParseDuration(c.Gradle.BuildCache.MaxAge); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", c.Gradle.BuildCache.MaxAge, err)
if g.MaxAge != "" && g.MaxAge != "0" {
if _, err := time.ParseDuration(g.MaxAge); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err)
}
}

// Validate Gradle max size if specified.
if c.Gradle.BuildCache.MaxSize != "" {
if _, err := ParseSize(c.Gradle.BuildCache.MaxSize); err != nil {
if g.MaxSize != "" {
if _, err := ParseSize(g.MaxSize); err != nil {
return fmt.Errorf("invalid gradle.build_cache.max_size: %w", err)
}
}

// Validate Gradle sweep interval if specified.
if c.Gradle.BuildCache.SweepInterval != "" {
d, err := time.ParseDuration(c.Gradle.BuildCache.SweepInterval)
if g.SweepInterval != "" {
d, err := time.ParseDuration(g.SweepInterval)
if err != nil {
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: %w", c.Gradle.BuildCache.SweepInterval, err)
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: %w", g.SweepInterval, err)
}
if d <= 0 {
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: must be > 0", c.Gradle.BuildCache.SweepInterval)
return fmt.Errorf("invalid gradle.build_cache.sweep_interval %q: must be > 0", g.SweepInterval)
}
}

return nil
}

const defaultGradleBuildCacheMaxUploadSize = 100 << 20
const defaultGradleBuildCacheSweepInterval = 10 * time.Minute
const (
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default
defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default
defaultGradleBuildCacheMaxUploadSize = 100 << 20
defaultGradleBuildCacheSweepInterval = 10 * time.Minute
defaultGradleMaxUploadSizeStr = "100MB"
defaultGradleSweepIntervalStr = "10m"
)

// ParseMaxSize returns the maximum cache size in bytes.
Expand Down
28 changes: 16 additions & 12 deletions internal/database/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ import (
"time"
)

const postgresTimestamp = "TIMESTAMP"
const (
postgresTimestamp = "TIMESTAMP"
sqliteDatetime = "DATETIME"
colTypeText = "TEXT"
)

// Schema for proxy-specific tables. The packages and versions tables
// are compatible with git-pkgs, allowing the proxy to use an existing
Expand Down Expand Up @@ -369,9 +373,9 @@ func isTableNotFound(err error) bool {
func (db *DB) createMigrationsTable() error {
var ts string
if db.dialect == DialectPostgres {
ts = "TIMESTAMP"
ts = postgresTimestamp
} else {
ts = "DATETIME"
ts = sqliteDatetime
}

query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS migrations (
Expand Down Expand Up @@ -457,12 +461,12 @@ func (db *DB) MigrateSchema() error {

func migrateAddPackagesEnrichmentColumns(db *DB) error {
columns := map[string]string{
"registry_url": "TEXT",
"supplier_name": "TEXT",
"supplier_type": "TEXT",
"source": "TEXT",
"enriched_at": "DATETIME",
"vulns_synced_at": "DATETIME",
"registry_url": colTypeText,
"supplier_name": colTypeText,
"supplier_type": colTypeText,
"source": colTypeText,
"enriched_at": sqliteDatetime,
"vulns_synced_at": sqliteDatetime,
}

if db.dialect == DialectPostgres {
Expand All @@ -487,10 +491,10 @@ func migrateAddPackagesEnrichmentColumns(db *DB) error {

func migrateAddVersionsEnrichmentColumns(db *DB) error {
columns := map[string]string{
"integrity": "TEXT",
"integrity": colTypeText,
"yanked": "INTEGER DEFAULT 0",
"source": "TEXT",
"enriched_at": "DATETIME",
"source": colTypeText,
"enriched_at": sqliteDatetime,
}

if db.dialect == DialectPostgres {
Expand Down
4 changes: 2 additions & 2 deletions internal/handler/conda.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) {
http.Error(w, "failed to create request", http.StatusInternalServerError)
return
}
req.Header.Set("Accept-Encoding", "gzip")
req.Header.Set(headerAcceptEncoding, "gzip")

resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
Expand Down Expand Up @@ -241,5 +241,5 @@ func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) {

// proxyUpstream forwards a request to Anaconda without caching.
func (h *CondaHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"})
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{headerAcceptEncoding})
}
2 changes: 1 addition & 1 deletion internal/handler/cran.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,5 +159,5 @@ func (h *CRANHandler) proxyCached(w http.ResponseWriter, r *http.Request) {

// proxyUpstream forwards a request to CRAN without caching.
func (h *CRANHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{"Accept-Encoding"})
h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, []string{headerAcceptEncoding})
}
4 changes: 2 additions & 2 deletions internal/handler/gem.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ func (h *GemHandler) fetchCompactIndex(r *http.Request, name string) (*http.Resp
if err != nil {
return nil, err
}
for _, hdr := range []string{"Accept", "Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
for _, hdr := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} {
if v := r.Header.Get(hdr); v != "" {
req.Header.Set(hdr, v)
}
Expand Down Expand Up @@ -311,7 +311,7 @@ func (h *GemHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
}

// Copy relevant headers
for _, h := range []string{"Accept", "Accept-Encoding", "If-None-Match", "If-Modified-Since"} {
for _, h := range []string{"Accept", headerAcceptEncoding, "If-None-Match", "If-Modified-Since"} {
if v := r.Header.Get(h); v != "" {
req.Header.Set(h, v)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/handler/gradle.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func (h *GradleBuildCacheHandler) parseCacheKey(urlPath string) (string, int) {
return "", http.StatusBadRequest
}

if keyPath == "" || strings.Contains(keyPath, "/") {
if strings.Contains(keyPath, "/") {
return "", http.StatusNotFound
}

Expand Down
4 changes: 3 additions & 1 deletion internal/handler/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ const defaultHTTPTimeout = 30 * time.Second

const contentTypeJSON = "application/json"

const headerAcceptEncoding = "Accept-Encoding"

// maxMetadataSize is the maximum size of upstream metadata responses (100 MB).
// Package metadata (e.g. npm with many versions) can be large, but unbounded
// reads risk OOM if an upstream misbehaves.
Expand Down Expand Up @@ -726,7 +728,7 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst
}
req.Header.Set("Accept", accept)

for _, header := range []string{"Accept-Encoding", "If-Modified-Since", "If-None-Match"} {
for _, header := range []string{headerAcceptEncoding, "If-Modified-Since", "If-None-Match"} {
if v := r.Header.Get(header); v != "" {
req.Header.Set(header, v)
}
Expand Down
6 changes: 3 additions & 3 deletions internal/handler/nuget.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request
http.Error(w, "failed to create request", http.StatusInternalServerError)
return
}
req.Header.Set("Accept-Encoding", "gzip")
req.Header.Set(headerAcceptEncoding, "gzip")

resp, err := h.proxy.HTTPClient.Do(req)
if err != nil {
Expand Down Expand Up @@ -338,8 +338,8 @@ func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
}

// Copy accept-encoding for compression
if ae := r.Header.Get("Accept-Encoding"); ae != "" {
req.Header.Set("Accept-Encoding", ae)
if ae := r.Header.Get(headerAcceptEncoding); ae != "" {
req.Header.Set(headerAcceptEncoding, ae)
}

resp, err := h.proxy.HTTPClient.Do(req)
Expand Down
27 changes: 10 additions & 17 deletions internal/server/gradle_cache_eviction.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,7 @@ func sweepGradleBuildCache(
return 0, 0, nil
}

sort.Slice(entries, func(i, j int) bool {
iTime := entries[i].ModTime
jTime := entries[j].ModTime

switch {
case iTime.IsZero() && jTime.IsZero():
return entries[i].Path < entries[j].Path
case iTime.IsZero():
return true
case jTime.IsZero():
return false
case iTime.Equal(jTime):
return entries[i].Path < entries[j].Path
default:
return iTime.Before(jTime)
}
})
sortOldestFirst(entries)

deletedCount := 0
freedBytes := int64(0)
Expand Down Expand Up @@ -154,3 +138,12 @@ func sweepGradleBuildCache(

return deletedCount, freedBytes, nil
}

func sortOldestFirst(entries []storage.ObjectInfo) {
sort.Slice(entries, func(i, j int) bool {
if entries[i].ModTime.Equal(entries[j].ModTime) {
return entries[i].Path < entries[j].Path
}
return entries[i].ModTime.Before(entries[j].ModTime)
})
}