Configuration groundwork

aws/client.go

@@ -1,6 +1,7 @@
 package aws
 
 import (
+	"compress/gzip"
 	"context"
 	"encoding/xml"
 	"fmt"
@@ -150,15 +151,30 @@ func (c *Client) Updates(ctx context.Context) (io.ReadCloser, error) {
 			tf.Close()
 			return nil, err
 		}
+		gz, err := gzip.NewReader(tf)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create gzip reader: %v", err)
+		}
 
 		log.Debug().Msg("success")
-		return tf, nil
+		return &gzippedFile{
+			Reader: gz,
+			Closer: tf,
+		}, nil
 	}
 
 	log.Error().Msg("exhausted all mirrors")
 	return nil, fmt.Errorf("all update_info mirrors failed to return a response")
 }
 
+// gzippedFile implements io.ReadCloser by proxying calls to different
+// underlying implementations. This is used to make sure the file that backs the
+// downloaded security database has Close called on it.
+type gzippedFile struct {
+	io.Reader
+	io.Closer
+}
+
 func (c *Client) getMirrors(ctx context.Context, release Release) error {
 	var (
 		req *http.Request

aws/updater.go

@@ -1,12 +1,10 @@
 package aws
 
 import (
-	"compress/gzip"
 	"context"
 	"encoding/xml"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"time"
 
@@ -59,12 +57,6 @@ func (u *Updater) Fetch(ctx context.Context, fingerprint driver.Fingerprint) (io
 		return nil, "", fmt.Errorf("failed to retrieve update info: %v", err)
 	}
 
-	gzip, err := gzip.NewReader(rc)
-	if err != nil {
-		return nil, "", fmt.Errorf("failed to create gzip reader: %v", err)
-	}
-	rc = ioutil.NopCloser(gzip)
-
 	return rc, driver.Fingerprint(updatesRepoMD.Checksum.Sum), nil
 }
 

internal/indexer/ecosystem.go

@@ -1,6 +1,10 @@
 package indexer
 
-import "context"
+import (
+	"context"
+
+	"github.com/rs/zerolog"
+)
 
 // Ecosystems group together scanners and a Coalescer which are commonly used together.
 //
@@ -17,7 +21,11 @@ type Ecosystem struct {
 }
 
 // EcosystemsToScanners extracts and dedupes multiple ecosystems and returns their discrete scanners
-func EcosystemsToScanners(ctx context.Context, ecosystems []*Ecosystem) ([]PackageScanner, []DistributionScanner, []RepositoryScanner, error) {
+func EcosystemsToScanners(ctx context.Context, ecosystems []*Ecosystem, disallowRemote bool) ([]PackageScanner, []DistributionScanner, []RepositoryScanner, error) {
+	log := zerolog.Ctx(ctx).With().
+		Str("component", "internal/indexer/EcosystemsToScanners").
+		Logger()
+	ctx = log.WithContext(ctx)
 	ps := []PackageScanner{}
 	ds := []DistributionScanner{}
 	rs := []RepositoryScanner{}
@@ -29,32 +37,56 @@ func EcosystemsToScanners(ctx context.Context, ecosystems []*Ecosystem) ([]Packa
 			return nil, nil, nil, err
 		}
 		for _, s := range pscanners {
-			if _, ok := seen[s.Name()]; !ok {
-				ps = append(ps, s)
-				seen[s.Name()] = struct{}{}
+			n := s.Name()
+			if _, ok := seen[n]; ok {
+				continue
+			}
+			seen[n] = struct{}{}
+			if _, ok := s.(RPCScanner); ok && disallowRemote {
+				log.Info().
+					Str("scanner", n).
+					Msg("disallowed by configuration")
+				continue
 			}
+			ps = append(ps, s)
 		}
 
 		dscanners, err := ecosystem.DistributionScanners(ctx)
 		if err != nil {
 			return nil, nil, nil, err
 		}
 		for _, s := range dscanners {
-			if _, ok := seen[s.Name()]; !ok {
-				ds = append(ds, s)
-				seen[s.Name()] = struct{}{}
+			n := s.Name()
+			if _, ok := seen[n]; ok {
+				continue
 			}
+			seen[n] = struct{}{}
+			if _, ok := s.(RPCScanner); ok && disallowRemote {
+				log.Info().
+					Str("scanner", n).
+					Msg("disallowed by configuration")
+				continue
+			}
+			ds = append(ds, s)
 		}
 
 		rscanners, err := ecosystem.RepositoryScanners(ctx)
 		if err != nil {
 			return nil, nil, nil, err
 		}
 		for _, s := range rscanners {
-			if _, ok := seen[s.Name()]; !ok {
-				rs = append(rs, s)
-				seen[s.Name()] = struct{}{}
+			n := s.Name()
+			if _, ok := seen[n]; ok {
+				continue
+			}
+			seen[n] = struct{}{}
+			if _, ok := s.(RPCScanner); ok && disallowRemote {
+				log.Info().
+					Str("scanner", n).
+					Msg("disallowed by configuration")
+				continue
 			}
+			rs = append(rs, s)
 		}
 	}
 	return ps, ds, rs, nil