Merge 9128553 into 143bc06

README.md

@@ -175,12 +175,12 @@ cost of running some static analysis tools.
 ### Usage
 
 Command line usage of `scc` is designed to be as simple as possible.
-Full details can be found in `scc --help` or `scc -h`.
+Full details can be found in `scc --help` or `scc -h`. Note that the below reflects the state of master not a release.
 
 ```
 $ scc -h
 Sloc, Cloc and Code. Count lines of code in a directory with complexity estimation.
-Version 2.13.0
+Version 3.0.0 (beta)
 Ben Boyter <ben@boyter.org> + Contributors
 
 Usage:
@@ -195,7 +195,7 @@ Flags:
       --debug                       enable debug output
       --exclude-dir strings         directories to exclude (default [.git,.hg,.svn])
       --file-gc-count int           number of files to parse before turning the GC on (default 10000)
-  -f, --format string               set output format [tabular, wide, json, csv, cloc-yaml, html, html-table] (default "tabular")
+  -f, --format string               set output format [tabular, wide, json, csv, cloc-yaml, html, html-table, sql, sql-insert] (default "tabular")
       --format-multi string         have multiple format output overriding --format [e.g. tabular:stdout,csv:file.csv,json:file.json]
       --gen                         identify generated files
       --generated-markers strings   string markers in head of generated files (default [do not edit])
@@ -224,6 +224,7 @@ Flags:
       --remap-unknown string        inspect files of unknown type and remap by checking for a string and remapping the language [e.g. "-*- C++ -*-":"C Header"]
       --size-unit string            set size unit [si, binary, mixed, xkcd-kb, xkcd-kelly, xkcd-imaginary, xkcd-intel, xkcd-drive, xkcd-bakers] (default "si")
   -s, --sort string                 column to sort by [files, name, lines, blanks, code, comments, complexity] (default "files")
+      --sql-project string          use supplied name as the project identifier for the current run. Only valid with the --format sql or sql-insert option
   -t, --trace                       enable trace output (not recommended when processing multiple files)
   -v, --verbose                     verbose output
       --version                     version for scc

SCC-OUTPUT-REPORT.html

@@ -12,12 +12,12 @@
 	<tbody><tr>
 		<th>Go</th>
 		<th>34</th>
-		<th>8088</th>
-		<th>1323</th>
-		<th>347</th>
-		<th>6418</th>
-		<th>1338</th>
-		<th>321289</th>
+		<th>8208</th>
+		<th>1340</th>
+		<th>349</th>
+		<th>6519</th>
+		<th>1352</th>
+		<th>324777</th>
 	</tr><tr>
 		<th>Java</th>
 		<th>24</th>
@@ -48,12 +48,12 @@
 	</tr><tr>
 		<th>Markdown</th>
 		<th>8</th>
-		<th>1133</th>
-		<th>265</th>
+		<th>1156</th>
+		<th>270</th>
 		<th>0</th>
-		<th>868</th>
+		<th>886</th>
 		<th>0</th>
-		<th>46552</th>
+		<th>47210</th>
 	</tr><tr>
 		<th>CSS</th>
 		<th>5</th>
@@ -93,12 +93,12 @@
 	</tr><tr>
 		<th>Shell</th>
 		<th>3</th>
-		<th>950</th>
-		<th>132</th>
+		<th>987</th>
+		<th>136</th>
 		<th>37</th>
-		<th>781</th>
-		<th>87</th>
-		<th>33879</th>
+		<th>814</th>
+		<th>91</th>
+		<th>35251</th>
 	</tr><tr>
 		<th>JavaServer Pages</th>
 		<th>2</th>
@@ -553,11 +553,11 @@
 	<tfoot><tr>
 		<th>Total</th>
 		<th>162</th>
-		<th>24483</th>
-		<th>2791</th>
-		<th>1540</th>
-		<th>20152</th>
-		<th>2258</th>
-    	<th>1738364</th>
+		<th>24663</th>
+		<th>2817</th>
+		<th>1542</th>
+		<th>20304</th>
+		<th>2276</th>
+    	<th>1743882</th>
 	</tr></tfoot>
 	</table></body></html>

main.go

@@ -93,7 +93,7 @@ func main() {
 		"format",
 		"f",
 		"tabular",
-		"set output format [tabular, wide, json, csv, cloc-yaml, html, html-table]",
+		"set output format [tabular, wide, json, csv, cloc-yaml, html, html-table, sql, sql-insert]",
 	)
 	flags.StringSliceVarP(
 		&processor.AllowListExtensions,
@@ -271,6 +271,12 @@ func main() {
 		"",
 		"have multiple format output overriding --format [e.g. tabular:stdout,csv:file.csv,json:file.json]",
 	)
+	flags.StringVar(
+		&processor.SQLProject,
+		"sql-project",
+		"",
+		"use supplied name as the project identifier for the current run. Only valid with the --format sql or sql-insert option",
+	)
 	flags.StringVar(
 		&processor.RemapUnknown,
 		"remap-unknown",

processor/formatters.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"math"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 	"time"
@@ -411,6 +412,66 @@ func toHtmlTable(input chan *FileJob) string {
 	return str.String()
 }
 
+func toSqlInsert(input chan *FileJob) string {
+	var str strings.Builder
+	projectName := SQLProject
+	if projectName == "" {
+		projectName = strings.Join(DirFilePaths, ",")
+	}
+
+	str.WriteString("\nbegin transaction;")
+	count := 0
+	for res := range input {
+		count++
+
+		dir, _ := filepath.Split(res.Location)
+
+		str.WriteString(fmt.Sprintf("\ninsert into t values('%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, %d);",
+			projectName, res.Language, res.Location, dir, res.Filename, res.Bytes, res.Blank, res.Comment, res.Code, res.Complexity))
+
+		// every 1000 files commit and start a new transaction to avoid overloading
+		if count == 1000 {
+			str.WriteString("\ncommit;")
+			str.WriteString("\nbegin transaction;")
+			count = 0
+		}
+	}
+	if count != 1000 {
+		str.WriteString("\ncommit;")
+	}
+
+	currentTime := time.Now()
+	es := float64(makeTimestampMilli()-startTimeMilli) * 0.001
+	str.WriteString("\nbegin transaction;")
+	str.WriteString(fmt.Sprintf("\ninsert into metadata values('%s', '%s', %f);", currentTime.Format("2006-01-02 15:04:05"), projectName, es))
+	str.WriteString("\ncommit;")
+
+	return str.String()
+}
+
+func toSql(input chan *FileJob) string {
+	var str strings.Builder
+
+	str.WriteString(`create table metadata (   -- github.com/boyter/scc v ` + Version + `
+             timestamp text,
+             Project   text,
+             elapsed_s real);
+create table t        (
+             Project       text   ,
+             Language      text   ,
+             File          text   ,
+             File_dirname  text   ,
+             File_basename text   ,
+             nByte         integer,
+             nBlank        integer,
+             nComment      integer,
+             nCode         integer,
+             nComplexity   integer   );`)
+
+	str.WriteString(toSqlInsert(input))
+	return str.String()
+}
+
 func fileSummarize(input chan *FileJob) string {
 	if FormatMulti != "" {
 		return fileSummarizeMulti(input)
@@ -429,6 +490,10 @@ func fileSummarize(input chan *FileJob) string {
 		return toHtml(input)
 	case strings.ToLower(Format) == "html-table":
 		return toHtmlTable(input)
+	case strings.ToLower(Format) == "sql":
+		return toSql(input)
+	case strings.ToLower(Format) == "sql-insert":
+		return toSqlInsert(input)
 	}
 
 	return fileSummarizeShort(input)
@@ -476,6 +541,10 @@ func fileSummarizeMulti(input chan *FileJob) string {
 				val = toHtml(i)
 			case "html-table":
 				val = toHtmlTable(i)
+			case "sql":
+				val = toSql(i)
+			case "sql-insert":
+				val = toSqlInsert(i)
 			}
 
 			if t[1] == "stdout" {

processor/formatters_test.go

@@ -606,6 +606,48 @@ func TestToCsvMultiple(t *testing.T) {
 	}
 }
 
+func TestToSQLSingle(t *testing.T) {
+	inputChan := make(chan *FileJob, 1000)
+	inputChan <- &FileJob{
+		Language:           "Go",
+		Filename:           "bbbb.go",
+		Extension:          "go",
+		Location:           "./",
+		Bytes:              1000,
+		Lines:              1000,
+		Code:               1000,
+		Comment:            1000,
+		Blank:              1000,
+		Complexity:         1000,
+		WeightedComplexity: 1000,
+		Binary:             false,
+	}
+	close(inputChan)
+	Debug = true // Increase coverage slightly
+	res := toSql(inputChan)
+	Debug = false
+
+	if !strings.Contains(res, `create table metadata`) {
+		t.Error("Expected create table return", res)
+	}
+
+	if !strings.Contains(res, `create table t`) {
+		t.Error("Expected create table return", res)
+	}
+
+	if !strings.Contains(res, `begin transaction`) {
+		t.Error("Expected begin transaction return", res)
+	}
+
+	if !strings.Contains(res, `insert into t values('', 'Go', './', './', 'bbbb.go', 1000, 1000, 1000, 1000, 1000);`) {
+		t.Error("Expected insert return", res)
+	}
+
+	if !strings.Contains(res, `insert into metadata values`) {
+		t.Error("Expected insert return", res)
+	}
+}
+
 func TestFileSummarizeWide(t *testing.T) {
 	inputChan := make(chan *FileJob, 1000)
 	inputChan <- &FileJob{

processor/processor.go

@@ -15,7 +15,7 @@ import (
 )
 
 // Version indicates the version of the application
-var Version = "2.13.0"
+var Version = "3.0.0 (beta)"
 
 // Flags set via the CLI which control how the output is displayed
 
@@ -103,6 +103,9 @@ var Format = ""
 // FormatMulti is a rule for defining multiple output formats
 var FormatMulti = ""
 
+// SQLProject is used to store the name for the SQL insert formats but is optional
+var SQLProject = ""
+
 // RemapUnknown allows remapping of unknown files with a string to search the content for
 var RemapUnknown = ""
 

test-all.sh

@@ -507,6 +507,24 @@ else
     exit
 fi
 
+if ./scc --format sql | grep -q "create table metadata"; then
+    echo -e "${GREEN}PASSED sql output test"
+else
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Should be able to output to sql"
+    echo -e "=======================================================${NC}"
+    exit
+fi
+
+if ./scc --format sql-insert | grep -q "insert into t values"; then
+    echo -e "${GREEN}PASSED sql-insert output test"
+else
+    echo -e "${RED}======================================================="
+    echo -e "FAILED Should be able to output to sql-insert"
+    echo -e "=======================================================${NC}"
+    exit
+fi
+
 if ./scc ./examples/countas/ --count-as jsp:html | grep -q "HTML"; then
     echo -e "${GREEN}PASSED counted JSP as HTML"
 else
@@ -624,7 +642,16 @@ else
     exit
 fi
 
-if ./scc --format-multi "tabular:stdout,html:stdout,csv:stdout" | grep -q "meta charset"; then
+if ./scc --format-multi "tabular:stdout,html:stdout,csv:stdout,sql:stdout" | grep -q "meta charset"; then
+    echo -e "${GREEN}PASSED format multi check"
+else
+    echo -e "${RED}======================================================="
+    echo -e "FAILED format multi check"
+    echo -e "=======================================================${NC}"
+    exit
+fi
+
+if ./scc --format-multi "tabular:stdout,html:stdout,csv:stdout,sql:stdout" | grep -q "insert into t values"; then
     echo -e "${GREEN}PASSED format multi check"
 else
     echo -e "${RED}======================================================="
@@ -651,7 +678,7 @@ else
     exit
 fi
 
-./scc --format-multi "tabular:output.tab,wide:output.wide,json:output.json,csv:output.csv,cloc-yaml:output.yaml,html:output.html,html-table:output.html2"
+./scc --format-multi "tabular:output.tab,wide:output.wide,json:output.json,csv:output.csv,cloc-yaml:output.yaml,html:output.html,html-table:output.html2,sql:output.sql"
 
 if test -f output.tab; then
     echo -e "${GREEN}PASSED output.tab check"
@@ -707,6 +734,15 @@ else
     exit
 fi
 
+if test -f output.sql; then
+    echo -e "${GREEN}PASSED output.sql check"
+else
+    echo -e "${RED}======================================================="
+    echo -e "FAILED output.sql check"
+    echo -e "=======================================================${NC}"
+    exit
+fi
+
 # Try out specific languages
 for i in 'Bosque ' 'Flow9 ' 'Bitbucket Pipeline ' 'Docker ignore ' 'Q# ' 'Futhark ' 'Alloy ' 'Wren ' 'Monkey C ' 'Alchemist ' 'Luna ' 'ignore ' 'XML Schema ' 'Web Services' 'Go ' 'Java ' 'Boo ' 'License ' 'BASH ' 'C Shell ' 'Korn Shell ' 'Makefile ' 'Shell ' 'Zsh ' 'Rakefile ' 'Gemfile ' 'Dockerfile ' 'Yarn ' 'Sieve ' 'F# ' 'Elm ' 'Terraform ' 'Clojure ' 'C# ' 'LLVM IR ' 'HAML '
 do
@@ -747,6 +783,7 @@ rm ./output.csv
 rm ./output.yaml
 rm ./output.html
 rm ./output.html2
+rm ./output.sql
 
 
 echo -e "${GREEN}================================================="