Mscasso/feature/add db config

CHANGELOG.md

@@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Upcoming changes...
+## [1.4.4] - 2025-08-28
+### Added
+- Added option to select scanning KB.
+  - `SCAN_KB_NAME` can be set to the scanning KB name. Default: oss. 
+- Updated dependency versions
 
 ## [1.4.3] - 2025-02-04
 ### Added

go.mod

@@ -1,56 +1,55 @@
 module scanoss.com/go-api
 
-go 1.22.0
+go 1.24.0
 
-toolchain go1.22.6
+toolchain go1.24.5
 
 require (
 	github.com/go-co-op/gocron v1.37.0
 	github.com/golobby/config/v3 v3.4.2
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/jpillora/ipfilter v1.2.9
-	github.com/scanoss/zap-logging-helper v0.3.2
-	github.com/stretchr/testify v1.10.0
-	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.59.0
-	go.opentelemetry.io/otel v1.34.0
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.34.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0
-	go.opentelemetry.io/otel/metric v1.34.0
-	go.opentelemetry.io/otel/sdk v1.34.0
-	go.opentelemetry.io/otel/sdk/metric v1.34.0
-	go.opentelemetry.io/otel/trace v1.34.0
+	github.com/scanoss/zap-logging-helper v0.4.0
+	github.com/stretchr/testify v1.11.1
+	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0
+	go.opentelemetry.io/otel v1.37.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0
+	go.opentelemetry.io/otel/metric v1.37.0
+	go.opentelemetry.io/otel/sdk v1.37.0
+	go.opentelemetry.io/otel/sdk/metric v1.37.0
+	go.opentelemetry.io/otel/trace v1.37.0
 	go.uber.org/zap v1.27.0
 )
 
 require (
-	github.com/BurntSushi/toml v1.3.2 // indirect
-	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/BurntSushi/toml v1.5.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golobby/cast v1.3.3 // indirect
 	github.com/golobby/dotenv v1.3.2 // indirect
 	github.com/golobby/env/v2 v2.2.4 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect
-	github.com/phuslu/iploc v1.0.20250131 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
+	github.com/phuslu/iploc v1.0.20250815 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/net v0.34.0 // indirect
-	golang.org/x/sys v0.29.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect
-	google.golang.org/grpc v1.70.0 // indirect
-	google.golang.org/protobuf v1.36.4 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect
+	google.golang.org/grpc v1.75.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

pkg/config/server_config.go

@@ -57,6 +57,7 @@ type ServerConfig struct {
 	Scanning struct {
 		WfpLoc         string `env:"SCAN_WFP_TMP"`            // specific location to write temporary WFP files to
 		ScanBinary     string `env:"SCAN_BINARY"`             // Binary to use for scanning
+		ScanKbName     string `env:"SCAN_KB_NAME"`            // KB name passed as "-n" parameter to the scanoss command
 		ScanDebug      bool   `env:"SCAN_DEBUG"`              // true/false
 		ScanFlags      int    `env:"SCAN_ENGINE_FLAGS"`       // Default flags to use when scanning
 		ScanTimeout    int    `env:"SCAN_ENGINE_TIMEOUT"`     // timeout for waiting for the scan engine to respond
@@ -106,6 +107,7 @@ func setServerConfigDefaults(cfg *ServerConfig) {
 	cfg.App.Mode = "dev"
 	cfg.Logging.DynamicPort = "localhost:60085"
 	cfg.Scanning.ScanBinary = "scanoss"
+	cfg.Scanning.ScanKbName = "oss"
 	cfg.Scanning.ScanFlags = 0
 	cfg.Scanning.TmpFileDelete = true
 	cfg.Scanning.Workers = 1       // Default to single threaded scanning

pkg/service/kb_details.go

@@ -85,7 +85,7 @@ func (s APIService) loadKBDetails() {
 		engineVersion = "unknown"
 	}
 	// Load a random (hopefully non-existent) file match to extract the KB version details
-	result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", "", "", zs)
+	result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", "", "", "", zs)
 	if err != nil {
 		zs.Warnf("Failed to detect KB version from eninge: %v", err)
 		return

pkg/service/scanning_service.go

@@ -86,7 +86,7 @@ func (s APIService) scanDirect(w http.ResponseWriter, r *http.Request, zs *zap.S
 		setSpanError(span, "No WFP contents supplied")
 		return 0
 	}
-	flags, scanType, sbom := s.getFlags(r, zs)
+	flags, scanType, sbom, dbName := s.getFlags(r, zs)
 	// Check if we have an SBOM (and type) supplied
 	var sbomFilename string
 	if len(sbom) > 0 && len(scanType) > 0 {
@@ -121,9 +121,9 @@ func (s APIService) scanDirect(w http.ResponseWriter, r *http.Request, zs *zap.S
 	s.countScanSize(wfps, wfpCount, zs, context, span)
 	// Only one worker selected, so send the whole WFP in a single command
 	if s.config.Scanning.Workers <= 1 {
-		s.singleScan(string(contentsTrimmed), flags, scanType, sbomFilename, zs, w)
+		s.singleScan(string(contentsTrimmed), flags, scanType, sbomFilename, dbName, zs, w)
 	} else {
-		s.scanThreaded(wfps, int(wfpCount), flags, scanType, sbomFilename, zs, w, span)
+		s.scanThreaded(wfps, int(wfpCount), flags, scanType, sbomFilename, dbName, zs, w, span)
 	}
 	return wfpCount
 }
@@ -155,10 +155,11 @@ func (s APIService) countScanSize(wfps []string, wfpCount int64, zs *zap.Sugared
 }
 
 // getFlags extracts the form values from a request returns the flags, scan type, and sbom data if detected.
-func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, string, string) {
-	flags := strings.TrimSpace(r.FormValue("flags"))   // Check form for Scanning flags
-	scanType := strings.TrimSpace(r.FormValue("type")) // Check form for SBOM type
-	sbom := strings.TrimSpace(r.FormValue("assets"))   // Check form for SBOM contents
+func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, string, string, string) {
+	flags := strings.TrimSpace(r.FormValue("flags"))    // Check form for Scanning flags
+	scanType := strings.TrimSpace(r.FormValue("type"))  // Check form for SBOM type
+	sbom := strings.TrimSpace(r.FormValue("assets"))    // Check form for SBOM contents
+	dbName := strings.TrimSpace(r.FormValue("db_name")) // Check form for db name
 	// TODO is it necessary to check the header also for these values?
 	if len(flags) == 0 {
 		flags = strings.TrimSpace(r.Header.Get("flags")) // Check header for Scanning flags
@@ -169,10 +170,13 @@ func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, st
 	if len(sbom) == 0 {
 		sbom = strings.TrimSpace(r.Header.Get("assets")) // Check header for SBOM contents
 	}
+	if len(dbName) == 0 {
+		dbName = strings.TrimSpace(r.Header.Get("db_name")) // Check header for SBOM contents
+	}
 	if s.config.App.Trace {
-		zs.Debugf("Header: %v, Form: %v, flags: %v, type: %v, assets: %v", r.Header, r.Form, flags, scanType, sbom)
+		zs.Debugf("Header: %v, Form: %v, flags: %v, type: %v, assets: %v, db_name %v", r.Header, r.Form, flags, scanType, sbom, dbName)
 	}
-	return flags, scanType, sbom
+	return flags, scanType, sbom, dbName
 }
 
 // writeSbomFile writes the given string into an SBOM temporary file.
@@ -192,9 +196,9 @@ func (s APIService) writeSbomFile(sbom string, zs *zap.SugaredLogger) (*os.File,
 }
 
 // singleScan runs a scan of the WFP in a single thread.
-func (s APIService) singleScan(wfp, flags, sbomType, sbomFile string, zs *zap.SugaredLogger, w http.ResponseWriter) {
+func (s APIService) singleScan(wfp, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger, w http.ResponseWriter) {
 	zs.Debugf("Single threaded scan...")
-	result, err := s.scanWfp(wfp, flags, sbomType, sbomFile, zs)
+	result, err := s.scanWfp(wfp, flags, sbomType, sbomFile, dbName, zs)
 	if err != nil {
 		zs.Errorf("Engine scan failed: %v", err)
 		http.Error(w, "ERROR engine scan failed", http.StatusInternalServerError)
@@ -212,7 +216,7 @@ func (s APIService) singleScan(wfp, flags, sbomType, sbomFile string, zs *zap.Su
 }
 
 // scanThreaded scan the given WFPs in multiple threads.
-func (s APIService) scanThreaded(wfps []string, wfpCount int, flags, sbomType, sbomFile string, zs *zap.SugaredLogger, w http.ResponseWriter, span oteltrace.Span) {
+func (s APIService) scanThreaded(wfps []string, wfpCount int, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger, w http.ResponseWriter, span oteltrace.Span) {
 	addSpanEvent(span, "Started Scanning.")
 	numWorkers := s.config.Scanning.Workers
 	groupedWfps := wfpCount / s.config.Scanning.WfpGrouping
@@ -229,7 +233,7 @@ func (s APIService) scanThreaded(wfps []string, wfpCount int, flags, sbomType, s
 	zs.Debugf("Creating %v scanning workers...", numWorkers)
 	// Create workers
 	for i := 1; i <= numWorkers; i++ {
-		go s.workerScan(fmt.Sprintf("%d_%s", i, uuid.New().String()), requests, results, flags, sbomType, sbomFile, zs)
+		go s.workerScan(fmt.Sprintf("%d_%s", i, uuid.New().String()), requests, results, flags, sbomType, sbomFile, dbName, zs)
 	}
 	requestCount := 0 // Count the number of actual requests sent
 	var wfpRequests []string
@@ -304,7 +308,7 @@ func (s APIService) validateHPSM(contents []byte, zs *zap.SugaredLogger, w http.
 }
 
 // workerScan attempts to process all incoming scanning jobs and dumps the results into the subsequent results channel.
-func (s APIService) workerScan(id string, jobs <-chan string, results chan<- string, flags, sbomType, sbomFile string, zs *zap.SugaredLogger) {
+func (s APIService) workerScan(id string, jobs <-chan string, results chan<- string, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger) {
 	if s.config.App.Trace {
 		zs.Debugf("Starting up scanning worker: %v", id)
 	}
@@ -318,7 +322,7 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str
 			zs.Warnf("Nothing in the job request to scan. Ignoring")
 			results <- ""
 		} else {
-			result, err := s.scanWfp(job, flags, sbomType, sbomFile, zs)
+			result, err := s.scanWfp(job, flags, sbomType, sbomFile, dbName, zs)
 			if s.config.App.Trace {
 				zs.Debugf("scan result (%v): %v, %v", id, result, err)
 			}
@@ -343,7 +347,7 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str
 }
 
 // scanWfp run the scanoss engine scan of the supplied WFP.
-func (s APIService) scanWfp(wfp, flags, sbomType, sbomFile string, zs *zap.SugaredLogger) (string, error) {
+func (s APIService) scanWfp(wfp, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger) (string, error) {
 	if len(wfp) == 0 {
 		zs.Warnf("Nothing in the job request to scan. Ignoring")
 		return "", fmt.Errorf("no wfp supplied to scan. ignoring")
@@ -367,6 +371,11 @@ func (s APIService) scanWfp(wfp, flags, sbomType, sbomFile string, zs *zap.Sugar
 	if s.config.Scanning.ScanDebug {
 		args = append(args, "-d") // Set debug mode
 	}
+	if len(dbName) > 0 && dbName != "" { // we want to prefer request over the local config
+		args = append(args, fmt.Sprintf("-n%s", dbName))
+	} else if s.config.Scanning.ScanKbName != "" { // Set scanning KB name
+		args = append(args, fmt.Sprintf("-n%s", s.config.Scanning.ScanKbName))
+	}
 	if s.config.Scanning.ScanFlags > 0 { // Set system flags if enabled
 		args = append(args, fmt.Sprintf("-F %v", s.config.Scanning.ScanFlags))
 	} else if len(flags) > 0 && flags != "0" { // Set user supplied flags if enabled

test-support/scanoss.sh

@@ -51,6 +51,19 @@ if [ "$1" == "-l" ] || [ "$2" == "-l" ] || [ "$3" == "-l" ] ; then
   exit 0
 fi
 
+# Simulate invalid kb name
+for arg in "$@"; do
+  if [[ "$arg" == "-n"* ]]; then
+    # Extract everything after "-n"
+    scf=${arg#-n}
+    # Only show error if the value is NOT "oss"
+    if [[ "$scf" != "oss" ]]; then
+      echo "{Error: file and url tables must be present in $scf KB in order to proceed with the scan"
+      exit 1
+    fi
+  fi
+done
+
 # Simulate return a scan result
 if [ "$1" == "-w" ] || [ "$2" == "-w" ] || [ "$3" == "-w" ] || [ "$4" == "-w" ] || [ "$5" == "-w" ] || [ "$6" == "-w" ] || [ "$7" == "-w" ] || [ "$8" == "-w" ]; then
   for i in "$@"; do :; done

tests/scanning_test.go

@@ -18,10 +18,11 @@ package tests
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/suite"
 	"io"
 	"net/http"
 	"testing"
+
+	"github.com/stretchr/testify/suite"
 )
 
 type E2EScanningSuite struct {
@@ -42,6 +43,13 @@ func (s *E2EScanningSuite) TestScanning() {
 		extraFields map[string]string
 		want        int
 	}{
+		{
+			name:        "Test Invalid  KB name",
+			filename:    "../pkg/service/tests/fingers.wfp",
+			shortName:   "fingers.wfp",
+			extraFields: map[string]string{"db_name": "test_kb"},
+			want:        http.StatusInternalServerError,
+		},
 		{
 			name:        "Test Empty WFP",
 			filename:    "../pkg/service/tests/fingers-empty.wfp",