processor.go

// SPDX-License-Identifier: MIT OR Unlicense

package processor

import (
	"encoding/base64"
	"fmt"
	jsoniter "github.com/json-iterator/go"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"sync"
)

// Version indicates the version of the application
var Version = "3.0.0"

// Flags set via the CLI which control how the output is displayed

// Files indicates if there should be file output or not when formatting
var Files = false

// Languages indicates if the command line should print out the supported languages
var Languages = false

// Verbose enables verbose logging output
var Verbose = false

// Debug enables debug logging output
var Debug = false

// Trace enables trace logging output which is extremely verbose
var Trace = false

// Duplicates enables duplicate file detection
var Duplicates = false

// MinifiedGenerated enables minified/generated file detection
var MinifiedGenerated = false

// IgnoreMinifiedGenerate printing counts for minified/generated files
var IgnoreMinifiedGenerate = false

// MinifiedGeneratedLineByteLength number of bytes per average line to determine file is minified/generated
var MinifiedGeneratedLineByteLength = 255

// Minified enables minified file detection
var Minified = false

// IgnoreMinified ignore printing counts for minified files
var IgnoreMinified = false

// Generated enables generated file detection
var Generated = false

// GeneratedMarkers defines head markers for generated file detection
var GeneratedMarkers []string

// IgnoreGenerated ignore printing counts for generated files
var IgnoreGenerated = false

// Complexity toggles complexity calculation
var Complexity = false

// More enables wider output with more information in formatter
var More = false

// Cocomo toggles the COCOMO calculation
var Cocomo = false

// CocomoProjectType allows the flipping between project types which impacts the calculation
var CocomoProjectType = "organic"

// Size toggles the Size calculation
var Size = false

// SizeUnit determines what size calculation is used for megabytes
var SizeUnit = "si"

// Ci indicates if running inside a CI so to disable box drawing characters
var Ci = false

// GitIgnore disables .gitignore checks
var GitIgnore = false

// Ignore disables ignore file checks
var Ignore = false

// DisableCheckBinary toggles checking for binary files using NUL bytes
var DisableCheckBinary = false

// SortBy sets which column output in formatter should be sorted by
var SortBy = ""

// Exclude is a regular expression which is used to exclude files from being processed
var Exclude = []string{}

// CountAs is a rule for mapping known or new extensions to other rules
var CountAs = ""

// Format sets the output format of the formatter
var Format = ""

// FormatMulti is a rule for defining multiple output formats
var FormatMulti = ""

// SQLProject is used to store the name for the SQL insert formats but is optional
var SQLProject = ""

// RemapUnknown allows remapping of unknown files with a string to search the content for
var RemapUnknown = ""

// RemapAll allows remapping of all files with a string to search the content for
var RemapAll = ""

// FileOutput sets the file that output should be written to
var FileOutput = ""

// PathDenyList sets the paths that should be skipped
var PathDenyList = []string{}

// DirectoryWalkerJobWorkers is the number of workers which will walk the directory tree
var DirectoryWalkerJobWorkers = runtime.NumCPU()

// FileListQueueSize is the queue of files found and ready to be read into memory
var FileListQueueSize = runtime.NumCPU()

// FileProcessJobWorkers is the number of workers that process the file collecting stats
var FileProcessJobWorkers = runtime.NumCPU() * 4

// FileSummaryJobQueueSize is the queue used to hold processed file statistics before formatting
var FileSummaryJobQueueSize = runtime.NumCPU()

// AllowListExtensions is a list of extensions which are allowed to be processed
var AllowListExtensions = []string{}

// AverageWage is the average wage in dollars used for the COCOMO cost estimate
var AverageWage int64 = 56286

// GcFileCount is the number of files to process before turning the GC back on
var GcFileCount = 10000
var gcPercent = -1
var isLazy = false

// NoLarge if set true will ignore files over a certain number of lines or bytes
var NoLarge = false

// IncludeSymLinks if set true will count symlink files
var IncludeSymLinks = false

// LargeLineCount number of lines before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44
var LargeLineCount int64 = 40000

// LargeByteCount number of bytes before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44
var LargeByteCount int64 = 1000000

// DirFilePaths is not set via flags but by arguments following the flags for file or directory to process
var DirFilePaths = []string{}

// Raw languageDatabase loaded
var languageDatabase = map[string]Language{}

// ExtensionToLanguage is loaded from the JSON that is in constants.go
var ExtensionToLanguage = map[string][]string{}

// ShebangLookup loaded from the JSON in constants.go contains shebang lookups
var ShebangLookup = map[string][]string{}

// FilenameToLanguage similar to ExtensionToLanguage loaded from the JSON in constants.go
var FilenameToLanguage = map[string]string{}

// LanguageFeatures contains the processed languages from processLanguageFeature
var LanguageFeatures = map[string]LanguageFeature{}

// LanguageFeaturesMutex is the shared mutex used to control getting and setting of language features
// used rather than sync.Map because it turned out to be marginally faster
var LanguageFeaturesMutex = sync.Mutex{}

// Start time in milli seconds in case we want the total time
var startTimeMilli = makeTimestampMilli()

// ConfigureLimits configures ulimits where possible
var ConfigureLimits func()

// ConfigureGc needs to be set outside of ProcessConstants because it should only be enabled in command line
// mode https://github.com/boyter/scc/issues/32
func ConfigureGc() {
	gcPercent = debug.SetGCPercent(gcPercent)
}

// ConfigureLazy is a simple setter used to turn on lazy loading used only by command line
func ConfigureLazy(lazy bool) {
	isLazy = lazy
}

// ProcessConstants is responsible for setting up the language features based on the JSON file that is stored in constants
// Needs to be called at least once in order for anything to actually happen
func ProcessConstants() {
	languageDatabase = loadDatabase()

	startTime := makeTimestampNano()
	for name, value := range languageDatabase {
		for _, ext := range value.Extensions {
			ExtensionToLanguage[ext] = append(ExtensionToLanguage[ext], name)
		}

		for _, fname := range value.FileNames {
			FilenameToLanguage[fname] = name
		}

		if len(value.SheBangs) != 0 {
			ShebangLookup[name] = value.SheBangs
		}
	}

	// If we have anything in CountAs set it up now
	if len(CountAs) != 0 {
		setupCountAs()
	}

	if Trace {
		printTrace(fmt.Sprintf("nanoseconds build extension to language: %d", makeTimestampNano()-startTime))
	}

	// Configure COCOMO setting
	_, ok := projectType[strings.ToLower(CocomoProjectType)]
	if !ok {
		// lets see if we can turn it into a custom one
		spl := strings.Split(CocomoProjectType, ",")
		val := []float64{}
		if len(spl) == 5 {
			// lets try to convert to float if we can
			for i := 1; i < 5; i++ {
				f, err := strconv.ParseFloat(spl[i], 64)
				if err == nil {
					val = append(val, f)
				}
			}
		}

		if len(val) == 4 {
			projectType[CocomoProjectType] = val
		} else {
			// if nothing matches fall back to organic
			CocomoProjectType = "organic"
		}
	}

	// If lazy is set then we want to load in the features as we find them not in one go
	// however otherwise being used as a library so just load them all in
	if !isLazy {
		startTime = makeTimestampMilli()
		for name, value := range languageDatabase {
			processLanguageFeature(name, value)
		}

		if Trace {
			printTrace(fmt.Sprintf("milliseconds build language features: %d", makeTimestampMilli()-startTime))
		}
	} else {
		if Trace {
			printTrace("configured to lazy load language features")
		}
	}
}

// Configure and setup any count-as params the use has supplied
func setupCountAs() {
	for _, s := range strings.Split(CountAs, ",") {
		t := strings.Split(s, ":")
		if len(t) == 2 {

			identified := false

			// There are two cases here.
			// first is they provide the name e.g. "Cargo Lock"
			// second is that the user supplies the extension EG wsdl
			// we should support BOTH cases
			// always remember we only need to validate t[1] as that's the one
			// that tells us where we are trying to map

			// See if we can identify based on language name which is the most
			// reliable as the name should be unique
			for name := range languageDatabase {
				if strings.ToLower(name) == strings.ToLower(t[1]) {
					ExtensionToLanguage[strings.ToLower(t[0])] = []string{name}
					identified = true
					if Debug {
						printDebug(fmt.Sprintf("set to count extension: %s as language %s by language", t[0], name))
					}
				}
			}

			// If the above did not work, its a matter of extension match
			// note that this is less reliable as some languages share extensions
			if !identified {
				target, ok := ExtensionToLanguage[strings.ToLower(t[1])]

				if ok {
					ExtensionToLanguage[strings.ToLower(t[0])] = target

					if Debug {
						printDebug(fmt.Sprintf("set to count extension: %s as language %s by extension", t[0], target))
					}
				}
			}
		}
	}
}

// LoadLanguageFeature will load a single feature as requested given the name
func LoadLanguageFeature(loadName string) {
	if !isLazy {
		return
	}

	// Check if already loaded and if so return because we don't need to do it again
	LanguageFeaturesMutex.Lock()
	_, ok := LanguageFeatures[loadName]
	LanguageFeaturesMutex.Unlock()
	if ok {
		return
	}

	var name string
	var value Language

	for name, value = range languageDatabase {
		if name == loadName {
			break
		}
	}

	startTime := makeTimestampNano()
	processLanguageFeature(loadName, value)
	if Trace {
		printTrace(fmt.Sprintf("nanoseconds to build language %s features: %d", loadName, makeTimestampNano()-startTime))
	}
}

func processLanguageFeature(name string, value Language) {
	complexityTrie := &Trie{}
	slCommentTrie := &Trie{}
	mlCommentTrie := &Trie{}
	stringTrie := &Trie{}
	tokenTrie := &Trie{}

	var complexityMask uint64
	var singleLineCommentMask uint64
	var multiLineCommentMask uint64
	var stringMask uint64
	var processMask uint64

	for _, v := range value.ComplexityChecks {
		complexityMask |= BloomTable[v[0]]
		complexityTrie.Insert(TComplexity, []byte(v))
		if !Complexity {
			tokenTrie.Insert(TComplexity, []byte(v))
		}
	}
	if !Complexity {
		processMask |= complexityMask
	}

	for _, v := range value.LineComment {
		singleLineCommentMask |= BloomTable[v[0]]
		slCommentTrie.Insert(TSlcomment, []byte(v))
		tokenTrie.Insert(TSlcomment, []byte(v))
	}
	processMask |= singleLineCommentMask

	for _, v := range value.MultiLine {
		multiLineCommentMask |= BloomTable[v[0][0]]
		mlCommentTrie.InsertClose(TMlcomment, []byte(v[0]), []byte(v[1]))
		tokenTrie.InsertClose(TMlcomment, []byte(v[0]), []byte(v[1]))
	}
	processMask |= multiLineCommentMask

	for _, v := range value.Quotes {
		stringMask |= BloomTable[v.Start[0]]
		stringTrie.InsertClose(TString, []byte(v.Start), []byte(v.End))
		tokenTrie.InsertClose(TString, []byte(v.Start), []byte(v.End))
	}
	processMask |= stringMask

	LanguageFeaturesMutex.Lock()
	LanguageFeatures[name] = LanguageFeature{
		Complexity:            complexityTrie,
		MultiLineComments:     mlCommentTrie,
		SingleLineComments:    slCommentTrie,
		Strings:               stringTrie,
		Tokens:                tokenTrie,
		Nested:                value.NestedMultiLine,
		ComplexityCheckMask:   complexityMask,
		MultiLineCommentMask:  multiLineCommentMask,
		SingleLineCommentMask: singleLineCommentMask,
		StringCheckMask:       stringMask,
		ProcessMask:           processMask,
		Keywords:              value.Keywords,
		Quotes:                value.Quotes,
	}
	LanguageFeaturesMutex.Unlock()
}

func processFlags() {
	// If wide/more mode is enabled we want the complexity calculation
	// to happen regardless as that is the only purpose of the flag
	if More && Complexity {
		Complexity = false
	}

	// If ignore minified/generated is on ensure we turn on the code to calculate that
	if IgnoreMinifiedGenerate {
		MinifiedGenerated = true
		IgnoreMinified = true
		IgnoreGenerated = true
	}

	if MinifiedGenerated {
		Minified = true
		Generated = true
	}

	if IgnoreMinified {
		Minified = true
	}

	if IgnoreGenerated {
		Generated = true
	}

	if Debug {
		printDebug(fmt.Sprintf("Path Deny List: %v", PathDenyList))
		printDebug(fmt.Sprintf("Sort By: %s", SortBy))
		printDebug(fmt.Sprintf("White List: %v", AllowListExtensions))
		printDebug(fmt.Sprintf("Files Output: %t", Files))
		printDebug(fmt.Sprintf("Verbose: %t", Verbose))
		printDebug(fmt.Sprintf("Duplicates Detection: %t", Duplicates))
		printDebug(fmt.Sprintf("Complexity Calculation: %t", !Complexity))
		printDebug(fmt.Sprintf("Wide: %t", More))
		printDebug(fmt.Sprintf("Average Wage: %d", AverageWage))
		printDebug(fmt.Sprintf("Cocomo: %t", !Cocomo))
		printDebug(fmt.Sprintf("Minified/Generated Detection: %t/%t", Minified, Generated))
		printDebug(fmt.Sprintf("Ignore Minified/Generated: %t/%t", IgnoreMinified, IgnoreGenerated))
		printDebug(fmt.Sprintf("IncludeSymLinks: %t", IncludeSymLinks))
	}
}

func loadDatabase() map[string]Language {
	var database map[string]Language
	startTime := makeTimestampMilli()

	data, err := base64.StdEncoding.DecodeString(languages)
	if err != nil {
		panic(fmt.Sprintf("failed to base64 decode languages: %v", err))
	}

	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	if err := json.Unmarshal(data, &database); err != nil {
		panic(fmt.Sprintf("languages json invalid: %v", err))
	}

	if Trace {
		printTrace(fmt.Sprintf("milliseconds unmarshal: %d", makeTimestampMilli()-startTime))
	}

	return database
}

func printLanguages() {
	database := loadDatabase()
	var names []string

	for key := range database {
		names = append(names, key)
	}

	sort.Slice(names, func(i, j int) bool {
		return strings.Compare(strings.ToLower(names[i]), strings.ToLower(names[j])) < 0
	})

	for _, name := range names {
		fmt.Println(fmt.Sprintf("%s (%s)", name, strings.Join(append(database[name].Extensions, database[name].FileNames...), ",")))
	}
}

// Process is the main entry point of the command line it sets everything up and starts running
func Process() {
	if Languages {
		printLanguages()
		return
	}

	ProcessConstants()
	processFlags()

	// Clean up any invalid arguments before setting everything up
	if len(DirFilePaths) == 0 {
		DirFilePaths = append(DirFilePaths, ".")
	}

	// Check if the paths or files added exist and exit if not
	for _, f := range DirFilePaths {
		fpath := filepath.Clean(f)

		if _, err := os.Stat(fpath); os.IsNotExist(err) {
			fmt.Println("file or directory does not exist: " + fpath)
			os.Exit(1)
		}
	}

	SortBy = strings.ToLower(SortBy)

	if Debug {
		printDebug(fmt.Sprintf("NumCPU: %d", runtime.NumCPU()))
		printDebug(fmt.Sprintf("SortBy: %s", SortBy))
		printDebug(fmt.Sprintf("PathDenyList: %v", PathDenyList))
	}

	fileListQueue := make(chan *FileJob, FileListQueueSize)             // Files ready to be read from disk
	fileSummaryJobQueue := make(chan *FileJob, FileSummaryJobQueueSize) // Files ready to be summarised

	go func() {
		directoryWalker := NewDirectoryWalker(fileListQueue)

		for _, f := range DirFilePaths {
			err := directoryWalker.Start(f)
			if err != nil {
				fmt.Printf("failed to walk %s: %v", f, err)
				os.Exit(1)
			}
		}

		directoryWalker.Run()
	}()
	go fileProcessorWorker(fileListQueue, fileSummaryJobQueue)

	result := fileSummarize(fileSummaryJobQueue)

	if FileOutput == "" {
		fmt.Println(result)
	} else {
		_ = ioutil.WriteFile(FileOutput, []byte(result), 0600)
		fmt.Println("results written to " + FileOutput)
	}
}