NIGH-154 Default Config (#42)

* add default detectors when nightfall config missing/invalid

* fix autoimport spacing

* move default logic to nightfall_config

* set default max routines to 20, not actual max if input is missing/invalid

* pr updates

* move logic to set number of routines to nightfall_config

* use const instead of hard-coded val

* pr updates

* rm outdated log statements

internal/clients/diffreviewer/circleci/circleci_service.go

@@ -15,7 +15,6 @@ import (
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/gitdiff"
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
 	circlelogger "github.com/nightfallai/nightfall_code_scanner/internal/clients/logger/circle_logger"
-	"github.com/nightfallai/nightfall_code_scanner/internal/clients/nightfall"
 	"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/gitdiffintf"
 	"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/githubintf"
 	"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
@@ -100,7 +99,7 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
 		BaseSHA:    beforeCommitSha,
 		Head:       s.PrDetails.CommitSha,
 	}
-	nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName)
+	nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName, s.Logger)
 	if err != nil {
 		s.Logger.Error("Error getting Nightfall config file. Ensure you have a Nightfall config file located in the root of your repository at .nightfalldlp/config.json with at least one Detector enabled")
 		return nil, err
@@ -110,17 +109,10 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
 		s.Logger.Error(fmt.Sprintf("Error getting Nightfall API key. Ensure you have %s set in the Github secrets of the repo", NightfallAPIKeyEnvVar))
 		return nil, errors.New("missing env var for nightfall api key")
 	}
-
-	var maxNumberRoutines int
-	if nightfallConfig.MaxNumberRoutines < nightfall.MaxConcurrentRoutinesCap {
-		maxNumberRoutines = nightfallConfig.MaxNumberRoutines
-	} else {
-		maxNumberRoutines = nightfall.MaxConcurrentRoutinesCap
-	}
 	return &nightfallconfig.Config{
 		NightfallAPIKey:            nightfallAPIKey,
 		NightfallDetectors:         nightfallConfig.Detectors,
-		NightfallMaxNumberRoutines: maxNumberRoutines,
+		NightfallMaxNumberRoutines: nightfallConfig.MaxNumberRoutines,
 		TokenExclusionList:         nightfallConfig.TokenExclusionList,
 		FileInclusionList:          nightfallConfig.FileInclusionList,
 		FileExclusionList:          nightfallConfig.FileExclusionList,

internal/clients/diffreviewer/circleci/circleci_service_test.go

@@ -137,6 +137,7 @@ const testOwner = "alan20854"
 const testRepo = "TestRepo"
 const testPrUrl = "https://github.com/alan20854/CircleCiTest/pull/3"
 const testConfigFileName = "nightfall_test_config.json"
+const testEmptyConfigFileName = "nightfall_empty_test_config.json"
 const excludedCreditCardRegex = "4242-4242-4242-[0-9]{4}"
 const excludedApiToken = "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv"
 const excludedIPRegex = "^127\\."
@@ -187,7 +188,7 @@ func (c *circleCiTestSuite) TestLoadConfig() {
 	}
 
 	nightfallConfig, err := tp.cs.LoadConfig(testConfigFileName)
-	c.NoError(err, "Error in LoadConfig")
+	c.NoError(err, "Unexpected error in LoadConfig")
 	c.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
 }
 
@@ -211,6 +212,34 @@ func (c *circleCiTestSuite) TestLoadConfigMissingApiKey() {
 	)
 }
 
+func (c *circleCiTestSuite) TestLoadEmptyConfig() {
+	tp := c.initTestParams()
+	apiKey := "api-key"
+	apiDetector := nightfallAPI.API_KEY
+	cryptoDetector := nightfallAPI.CRYPTOGRAPHIC_TOKEN
+	workspace, err := os.Getwd()
+	c.NoError(err, "Error getting workspace")
+	workspacePath := path.Join(workspace, "../../../../test/data")
+	os.Setenv(WorkspacePathEnvVar, workspacePath)
+	os.Setenv(CircleCurrentCommitShaEnvVar, commitSha)
+	os.Setenv(CircleBeforeCommitEnvVar, prevCommitSha)
+	os.Setenv(CircleBranchEnvVar, testBranch)
+	os.Setenv(CircleOwnerNameEnvVar, testOwner)
+	os.Setenv(CircleRepoNameEnvVar, testRepo)
+	os.Setenv(CirclePullRequestUrlEnvVar, testPrUrl)
+	os.Setenv(NightfallAPIKeyEnvVar, apiKey)
+
+	expectedNightfallConfig := &nightfallconfig.Config{
+		NightfallAPIKey:            apiKey,
+		NightfallDetectors:         []*nightfallAPI.Detector{&apiDetector, &cryptoDetector},
+		NightfallMaxNumberRoutines: nightfallconfig.DefaultMaxNumberRoutines,
+	}
+
+	nightfallConfig, err := tp.cs.LoadConfig(testEmptyConfigFileName)
+	c.NoError(err, "Unexpected error in LoadConfig")
+	c.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
+}
+
 func (c *circleCiTestSuite) TestGetDiff() {
 	tp := c.initTestParams()
 	ctrl := gomock.NewController(c.T())

internal/clients/diffreviewer/github/github_service.go

@@ -9,14 +9,12 @@ import (
 	"os"
 	"strings"
 
-	"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer/diffutils"
-
 	"github.com/google/go-github/v31/github"
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer"
+	"github.com/nightfallai/nightfall_code_scanner/internal/clients/diffreviewer/diffutils"
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/gitdiff"
 	"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
 	githublogger "github.com/nightfallai/nightfall_code_scanner/internal/clients/logger/github_logger"
-	"github.com/nightfallai/nightfall_code_scanner/internal/clients/nightfall"
 	"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/gitdiffintf"
 	"github.com/nightfallai/nightfall_code_scanner/internal/interfaces/githubintf"
 	"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
@@ -189,7 +187,7 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
 		BaseSHA:    event.Before,
 		Head:       s.CheckRequest.SHA,
 	}
-	nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName)
+	nightfallConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, nightfallConfigFileName, s.Logger)
 	if err != nil {
 		s.Logger.Error("Error getting Nightfall config file. Ensure you have a Nightfall config file located in the root of your repository at .nightfalldlp/config.json with at least one Detector enabled")
 		return nil, err
@@ -199,16 +197,10 @@ func (s *Service) LoadConfig(nightfallConfigFileName string) (*nightfallconfig.C
 		s.Logger.Error(fmt.Sprintf("Error getting Nightfall API key. Ensure you have %s set in the Github secrets of the repo", NightfallAPIKeyEnvVar))
 		return nil, errors.New("Missing env var for nightfall api key")
 	}
-	var maxNumberRoutines int
-	if nightfallConfig.MaxNumberRoutines < nightfall.MaxConcurrentRoutinesCap {
-		maxNumberRoutines = nightfallConfig.MaxNumberRoutines
-	} else {
-		maxNumberRoutines = nightfall.MaxConcurrentRoutinesCap
-	}
 	return &nightfallconfig.Config{
 		NightfallAPIKey:            nightfallAPIKey,
 		NightfallDetectors:         nightfallConfig.Detectors,
-		NightfallMaxNumberRoutines: maxNumberRoutines,
+		NightfallMaxNumberRoutines: nightfallConfig.MaxNumberRoutines,
 		TokenExclusionList:         nightfallConfig.TokenExclusionList,
 		FileInclusionList:          nightfallConfig.FileInclusionList,
 		FileExclusionList:          nightfallConfig.FileExclusionList,

internal/clients/diffreviewer/github/github_service_test.go

@@ -138,6 +138,7 @@ func (g *githubTestSuite) initTestParams() *testParams {
 }
 
 const testConfigFileName = "nightfall_test_config.json"
+const testEmptyConfigFileName = "nightfall_test_empty_config.json"
 const excludedCreditCardRegex = "4242-4242-4242-[0-9]{4}"
 const excludedApiToken = "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv"
 const excludedIPRegex = "^127\\."
@@ -189,9 +190,44 @@ func (g *githubTestSuite) TestLoadConfig() {
 	}
 
 	nightfallConfig, err := tp.gc.LoadConfig(testConfigFileName)
-	g.NoError(err, "Error in LoadConfig")
+	g.NoError(err, "Unexpected error in LoadConfig")
 	g.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
-	g.Equal(expectedGithubCheckRequest, tp.gc.CheckRequest, "Incorrect nightfall config")
+	g.Equal(expectedGithubCheckRequest, tp.gc.CheckRequest, "Incorrect github check request")
+}
+
+func (g *githubTestSuite) TestLoadEmptyConfig() {
+	tp := g.initTestParams()
+	apiKey := "api-key"
+	sha := "1234"
+	owner := "nightfallai"
+	repo := "testRepo"
+	apiDetector := nightfallAPI.API_KEY
+	cryptoDetector := nightfallAPI.CRYPTOGRAPHIC_TOKEN
+	pullRequest := 1
+	workspace, err := os.Getwd()
+	g.NoError(err, "Error getting workspace")
+	workspacePath := path.Join(workspace, "../../../../test/data")
+	eventPath := path.Join(workspace, "../../../../test/data/github_action_event.json")
+	os.Setenv(githubservice.WorkspacePathEnvVar, workspacePath)
+	os.Setenv(githubservice.EventPathEnvVar, eventPath)
+	os.Setenv(githubservice.NightfallAPIKeyEnvVar, apiKey)
+
+	expectedNightfallConfig := &nightfallconfig.Config{
+		NightfallAPIKey:            apiKey,
+		NightfallDetectors:         []*nightfallAPI.Detector{&apiDetector, &cryptoDetector},
+		NightfallMaxNumberRoutines: nightfallconfig.DefaultMaxNumberRoutines,
+	}
+	expectedGithubCheckRequest := &githubservice.CheckRequest{
+		Owner:       owner,
+		Repo:        repo,
+		SHA:         sha,
+		PullRequest: pullRequest,
+	}
+
+	nightfallConfig, err := tp.gc.LoadConfig(testEmptyConfigFileName)
+	g.NoError(err, "Unexpected error in LoadConfig")
+	g.Equal(expectedNightfallConfig, nightfallConfig, "Incorrect nightfall config")
+	g.Equal(expectedGithubCheckRequest, tp.gc.CheckRequest, "Incorrect github check request")
 }
 
 func (g *githubTestSuite) TestGetDiff() {

internal/clients/nightfall/nightfall.go

@@ -31,8 +31,6 @@ const (
 	maxItemsForAPIReq = 479
 	// timeout for the total time spent sending scan requests and receiving responses for a diff
 	defaultTimeout = time.Minute * 20
-	// maximum number of routines (scan request + response) running at once
-	MaxConcurrentRoutinesCap = 50
 	// maximum attempts to Nightfall API upon receiving 429 Too Many Requests before failing
 	MaxScanAttempts = 5
 	// initial delay before re-attempting scan request

internal/nightfallconfig/nightfall_config.go

@@ -3,15 +3,23 @@ package nightfallconfig
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 
+	"github.com/nightfallai/nightfall_code_scanner/internal/clients/logger"
 	nightfallAPI "github.com/nightfallai/nightfall_go_client/generated"
 )
 
-const defaultMaxNumberRoutines = 30
+// maximum number of routines (scan request + response) running at once
+const MaxConcurrentRoutinesCap = 50
+const DefaultMaxNumberRoutines = 20
 const nightfallConfigFilename = ".nightfalldlp/config.json"
+const defaultDetectorsInfoMessage = "Using default detectors (API_KEY and CRYTOGRAPHIC_TOKEN)"
+
+var apiKeyDetector = nightfallAPI.API_KEY
+var cryptoKeyDetector = nightfallAPI.CRYPTOGRAPHIC_TOKEN
 
 // NightfallConfigFileStructure struct representation of nightfall config file
 type NightfallConfigFileStructure struct {
@@ -32,16 +40,24 @@ type Config struct {
 	FileExclusionList          []string
 }
 
-// GetNightfallConfigFile loads nightfall config from file
-func GetNightfallConfigFile(workspacePath, fileName string) (*NightfallConfigFileStructure, error) {
+// GetNightfallConfigFile loads nightfall config from file, returns default if missing/invalid
+func GetNightfallConfigFile(workspacePath, fileName string, logger logger.Logger) (*NightfallConfigFileStructure, error) {
+	defaultNightfallConfig := &NightfallConfigFileStructure{
+		Detectors:         []*nightfallAPI.Detector{&apiKeyDetector, &cryptoKeyDetector},
+		MaxNumberRoutines: DefaultMaxNumberRoutines,
+	}
 	nightfallConfigFile, err := os.Open(path.Join(workspacePath, fileName))
 	if err != nil {
-		return nil, err
+		logger.Warning(fmt.Sprintf("Error opening nightfall config: %s", err.Error()))
+		logger.Info(defaultDetectorsInfoMessage)
+		return defaultNightfallConfig, nil
 	}
 	defer nightfallConfigFile.Close()
 	byteValue, err := ioutil.ReadAll(nightfallConfigFile)
 	if err != nil {
-		return nil, err
+		logger.Warning(fmt.Sprintf("Error reading nightfall config: %s", err.Error()))
+		logger.Info(defaultDetectorsInfoMessage)
+		return defaultNightfallConfig, nil
 	}
 	var nightfallConfig NightfallConfigFileStructure
 	err = json.Unmarshal(byteValue, &nightfallConfig)
@@ -51,8 +67,10 @@ func GetNightfallConfigFile(workspacePath, fileName string) (*NightfallConfigFil
 	if len(nightfallConfig.Detectors) < 1 {
 		return nil, errors.New("Nightfall config file is missing detectors")
 	}
-	if nightfallConfig.MaxNumberRoutines == 0 {
-		nightfallConfig.MaxNumberRoutines = defaultMaxNumberRoutines
+	if nightfallConfig.MaxNumberRoutines <= 0 {
+		nightfallConfig.MaxNumberRoutines = DefaultMaxNumberRoutines
+	} else if nightfallConfig.MaxNumberRoutines > MaxConcurrentRoutinesCap {
+		nightfallConfig.MaxNumberRoutines = MaxConcurrentRoutinesCap
 	}
 	nightfallConfig.FileExclusionList = append(nightfallConfig.FileExclusionList, nightfallConfigFilename)
 	return &nightfallConfig, nil

internal/nightfallconfig/nightfall_config_test.go

@@ -5,12 +5,14 @@ import (
 	"path"
 	"testing"
 
+	githublogger "github.com/nightfallai/nightfall_code_scanner/internal/clients/logger/github_logger"
 	"github.com/nightfallai/nightfall_code_scanner/internal/nightfallconfig"
 	nightfallAPI "github.com/nightfallai/nightfall_go_client/generated"
 	"github.com/stretchr/testify/assert"
 )
 
 const testFileName = "nightfall_test_config.json"
+const testMissingFileName = "nightfall_test_missing_config.json"
 const excludedCreditCardRegex = "4242-4242-4242-[0-9]{4}"
 const excludedApiToken = "xG0Ct4Wsu3OTcJnE1dFLAQfRgL6b8tIv"
 const excludedIPRegex = "^127\\."
@@ -29,7 +31,22 @@ func TestGetNightfallConfig(t *testing.T) {
 		FileInclusionList:  []string{"*"},
 		FileExclusionList:  []string{".nightfalldlp/config.json"},
 	}
-	actualConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, testFileName)
-	assert.NoError(t, err, "Unexpected error when GetNightfallConfig")
+	actualConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, testFileName, nil)
+	assert.NoError(t, err, "Unexpected error in test GetNightfallConfig")
+	assert.Equal(t, expectedConfig, actualConfig, "Incorrect nightfall config")
+}
+
+func TestGetNightfallConfigMissingConfigFile(t *testing.T) {
+	apiDetector := nightfallAPI.API_KEY
+	cryptoDetector := nightfallAPI.CRYPTOGRAPHIC_TOKEN
+	workspaceConfig, err := os.Getwd()
+	assert.NoError(t, err, "Unexpected error when getting current directory")
+	workspacePath := path.Join(workspaceConfig, "../../test/data")
+	expectedConfig := &nightfallconfig.NightfallConfigFileStructure{
+		Detectors:         []*nightfallAPI.Detector{&apiDetector, &cryptoDetector},
+		MaxNumberRoutines: nightfallconfig.DefaultMaxNumberRoutines,
+	}
+	actualConfig, err := nightfallconfig.GetNightfallConfigFile(workspacePath, testMissingFileName, githublogger.NewDefaultGithubLogger())
+	assert.NoError(t, err, "Unexpected error in test GetNightfallConfigMissingConfigFile")
 	assert.Equal(t, expectedConfig, actualConfig, "Incorrect nightfall config")
 }