From 47678695e90d3cd01d356ba9819ff5bfc70dc882 Mon Sep 17 00:00:00 2001 From: Denis Budyak Date: Mon, 18 Feb 2019 12:53:45 +0300 Subject: [PATCH] Add support to count commits (#80) * count(*) works like '*' * count(*) kind of works, some tests are still missing * Added more tests * Fix typo * Don't apply limit when counting records (rembmer limit=0 is forbidden) --- lexical/lexemes.go | 1 + lexical/lexical.go | 2 ++ lexical/lexical_test.go | 10 ++++++-- lexical/tokens.go | 4 ++- parser/ast.go | 1 + parser/parser.go | 47 +++++++++++++++++++++++++++++++++-- parser/parser_test.go | 30 ++++++++++++++++++++++ runtime/commits.go | 25 +++++++++++++------ runtime/reference.go | 20 +++++++++++---- runtime/remotes.go | 20 +++++++++++---- runtime/runtime.go | 4 +++ runtime/runtime_test.go | 26 +++++++++++++++++++ runtime/visitor.go | 11 ++++++-- runtime/visitor_test.go | 9 +++++++ semantical/semantical.go | 2 +- semantical/semantical_test.go | 2 +- 16 files changed, 187 insertions(+), 27 deletions(-) diff --git a/lexical/lexemes.go b/lexical/lexemes.go index 1379c83..99d78e2 100644 --- a/lexical/lexemes.go +++ b/lexical/lexemes.go @@ -13,3 +13,4 @@ const L_ASC = "asc" const L_DESC = "desc" const L_LIKE = "like" const L_NOT = "not" +const L_COUNT = "count" diff --git a/lexical/lexical.go b/lexical/lexical.go index 8b3c3cb..486bde1 100644 --- a/lexical/lexical.go +++ b/lexical/lexical.go @@ -222,6 +222,8 @@ func lexemeToToken(lexeme string) uint8 { return T_LIKE case L_NOT: return T_NOT + case L_COUNT: + return T_COUNT } return T_ID } diff --git a/lexical/lexical_test.go b/lexical/lexical_test.go index fa89dc6..da05a9f 100644 --- a/lexical/lexical_test.go +++ b/lexical/lexical_test.go @@ -86,7 +86,7 @@ func TestRecognizeTokensWithLexemesOfTwoChars(t *testing.T) { func TestRecognizeTokensWithSourceManySpaced(t *testing.T) { setUp() - source = "= < >= != cloudson" + source = "= < >= != cloudson count" char = nextChar() var token uint8 @@ -105,6 +105,9 @@ func TestRecognizeTokensWithSourceManySpaced(t *testing.T) { token, _ = Token() assertToken(t, token, T_ID) + + token, _ = Token() + assertToken(t, token, T_COUNT) } func TestErrorUnrecognizeChar(t *testing.T) { @@ -127,7 +130,7 @@ func TestErrorUnrecognizeChar(t *testing.T) { func TestReservedWords(t *testing.T) { setUp() - source = "SELECT from WHEre in not" + source = "SELECT from WHEre in not cOuNt" char = nextChar() var token uint8 @@ -147,6 +150,9 @@ func TestReservedWords(t *testing.T) { token, _ = Token() assertToken(t, token, T_NOT) + token, _ = Token() + assertToken(t, token, T_COUNT) + token, _ = Token() assertToken(t, token, T_EOF) } diff --git a/lexical/tokens.go b/lexical/tokens.go index 6a51461..6762ad5 100644 --- a/lexical/tokens.go +++ b/lexical/tokens.go @@ -27,6 +27,7 @@ const T_IN = 24 const T_ASC = 25 const T_LIKE = 26 const T_NOT = 27 +const T_COUNT = 28 const T_EOF = 0 const T_FUCK = 66 @@ -59,7 +60,8 @@ func allocMapTokenNames() { T_IN: "T_IN", T_EOF: "T_EOF", T_ASC: "T_ASC", - T_NOT: "T_NOT", + T_NOT: "T_NOT", + T_COUNT: "T_COUNT", } } } diff --git a/parser/ast.go b/parser/ast.go index 0d61381..3e7dc28 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -23,6 +23,7 @@ type NodeProgram struct { type NodeSelect struct { WildCard bool + Count bool Fields []string Tables []string Where NodeExpr diff --git a/parser/parser.go b/parser/parser.go index 94d3d19..b02ea6a 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -74,6 +74,7 @@ func gSelect() (*NodeSelect, error) { return nil, throwSyntaxError(lexical.T_SELECT, look_ahead) } token, tokenError := lexical.Token() + look_ahead = token if tokenError != nil { return nil, tokenError @@ -86,8 +87,13 @@ func gSelect() (*NodeSelect, error) { return nil, err } - if len(fields) == 1 && fields[0] == "*" { - s.WildCard = true + if len(fields) == 1 { + f0 := fields[0] + if f0 == "*" { + s.WildCard = true + } else if f0 == "#" { + s.Count = true + } } s.Fields = fields @@ -159,6 +165,9 @@ func gTableParams() ([]string, error) { } look_ahead = token return []string{"*"}, nil + } else if look_ahead == lexical.T_COUNT { + result, err := gCount() + return result, err } var fields = []string{} if look_ahead == lexical.T_ID { @@ -173,7 +182,41 @@ func gTableParams() ([]string, error) { return fields, errorSyntax } return nil, throwSyntaxError(lexical.T_ID, look_ahead) +} + +// consume count(*) +func gCount() ([]string, error) { + // by construction, T_COUNT is consumed and stored + // in the look_ahead + err := gExactlyASpecificToken(lexical.T_COUNT) + if err != nil { + return nil, err + } + err = gExactlyASpecificToken(lexical.T_PARENTH_L) + if err != nil { + return nil, err + } + err = gExactlyASpecificToken(lexical.T_WILD_CARD) + if err != nil { + return nil, err + } + err = gExactlyASpecificToken(lexical.T_PARENTH_R) + if err != nil { + return nil, err + } + return []string{"#"}, nil +} +func gExactlyASpecificToken(expected uint8) error { + if look_ahead != expected { + return throwSyntaxError(expected, look_ahead) + } + token, err := lexical.Token() + if err != nil { + return err + } + look_ahead = token + return nil } func gTableParamsRest(fields *[]string, count int) ([]string, error) { diff --git a/parser/parser_test.go b/parser/parser_test.go index 0c4535b..77ce475 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -55,6 +55,25 @@ func TestUsingWildCard(t *testing.T) { } } +func TestUsingCount(t *testing.T) { + New("select count(*) from users") + ast, error := AST() + + if error != nil { + t.Errorf(error.Error()) + } + + if ast.Child == nil { + t.Errorf("Program is empty") + } + + selectNode := ast.Child.(*NodeSelect) + if !selectNode.Count { + t.Errorf("Expected count setted") + } +} + + func TestUsingOneFieldName(t *testing.T) { New("select name from files") @@ -119,6 +138,17 @@ func TestErrorWithUnexpectedComma(t *testing.T) { } } +func TestErrorWithMalformedCount(t *testing.T) { + New("select count(*]") + + _, error := AST() + + if error == nil { + t.Errorf("Expected error 'Expected )'") + } +} + + func TestErrorWithInvalidRootNode(t *testing.T) { New("name from files") diff --git a/runtime/commits.go b/runtime/commits.go index 91a2642..aaf3c5b 100644 --- a/runtime/commits.go +++ b/runtime/commits.go @@ -2,6 +2,7 @@ package runtime import ( "fmt" + "strconv" "log" "strings" @@ -26,7 +27,7 @@ func walkCommits(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, er resultFields := fields // These are the fields in output with wildcards expanded rows := make([]tableRow, s.Limit) usingOrder := false - if s.Order != nil { + if s.Order != nil && !s.Count { usingOrder = true // Check if the order by field is in the selected fields. If not, add them to selected fields list if !utilities.IsFieldPresentInArray(fields, s.Order.Field) { @@ -38,15 +39,16 @@ func walkCommits(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, er boolRegister = true visitor.VisitExpr(where) if boolRegister { - newRow := make(tableRow) - for _, f := range fields { - newRow[f] = metadataCommit(f, object) + if !s.Count { + newRow := make(tableRow) + for _, f := range fields { + newRow[f] = metadataCommit(f, object) + } + rows = append(rows, newRow) } - rows = append(rows, newRow) - counter = counter + 1 } - if !usingOrder && counter > s.Limit { + if !usingOrder && !s.Count && counter > s.Limit { return false } return true @@ -56,13 +58,20 @@ func walkCommits(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, er if err != nil { fmt.Printf(err.Error()) } + if s.Count { + newRow := make(tableRow) + // counter was started from 1! + newRow[COUNT_FIELD_NAME] = strconv.Itoa(counter-1) + counter = 2 + rows = append(rows, newRow) + } rowsSliced := rows[len(rows)-counter+1:] rowsSliced, err = orderTable(rowsSliced, s.Order) if err != nil { return nil, err } - if usingOrder && counter > s.Limit { + if usingOrder && !s.Count && counter > s.Limit { counter = s.Limit rowsSliced = rowsSliced[0:counter] } diff --git a/runtime/reference.go b/runtime/reference.go index e09fdbd..b2c4b25 100644 --- a/runtime/reference.go +++ b/runtime/reference.go @@ -1,6 +1,7 @@ package runtime import ( + "strconv" "log" "github.com/cloudson/git2go" @@ -23,7 +24,7 @@ func walkReferences(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, } rows := make([]tableRow, s.Limit) usingOrder := false - if s.Order != nil { + if s.Order != nil && !s.Count { usingOrder = true } for object, inTheEnd := iterator.Next(); inTheEnd == nil; object, inTheEnd = iterator.Next() { @@ -36,17 +37,26 @@ func walkReferences(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, if s.WildCard { fields = builder.possibleTables[s.Tables[0]] } - newRow := make(tableRow) - for _, f := range fields { - newRow[f] = metadataReference(f, object) + if !s.Count { + newRow := make(tableRow) + for _, f := range fields { + newRow[f] = metadataReference(f, object) + } + rows = append(rows, newRow) } - rows = append(rows, newRow) counter = counter + 1 if !usingOrder && counter > s.Limit { break } } } + if s.Count { + newRow := make(tableRow) + // counter was started from 1! + newRow[COUNT_FIELD_NAME] = strconv.Itoa(counter-1) + counter = 2 + rows = append(rows, newRow) + } rowsSliced := rows[len(rows)-counter+1:] rowsSliced, err = orderTable(rowsSliced, s.Order) if err != nil { diff --git a/runtime/remotes.go b/runtime/remotes.go index 91c348c..d1d88c7 100644 --- a/runtime/remotes.go +++ b/runtime/remotes.go @@ -1,6 +1,7 @@ package runtime import ( + "strconv" "log" "github.com/cloudson/git2go" @@ -24,7 +25,7 @@ func walkRemotes(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, er } rows := make([]tableRow, s.Limit) usingOrder := false - if s.Order != nil { + if s.Order != nil && !s.Count { usingOrder = true } for _, remoteName := range remoteNames { @@ -37,11 +38,13 @@ func walkRemotes(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, er boolRegister = true visitor.VisitExpr(where) if boolRegister { - newRow := make(map[string]interface{}) - for _, f := range fields { - newRow[f] = metadataRemote(f, object) + if !s.Count { + newRow := make(map[string]interface{}) + for _, f := range fields { + newRow[f] = metadataRemote(f, object) + } + rows = append(rows, newRow) } - rows = append(rows, newRow) counter = counter + 1 if !usingOrder && counter > s.Limit { @@ -49,6 +52,13 @@ func walkRemotes(n *parser.NodeProgram, visitor *RuntimeVisitor) (*TableData, er } } } + if s.Count { + newRow := make(tableRow) + // counter was started from 1! + newRow[COUNT_FIELD_NAME] = strconv.Itoa(counter-1) + counter = 2 + rows = append(rows, newRow) + } rowsSliced := rows[len(rows)-counter+1:] rowsSliced, err = orderTable(rowsSliced, s.Order) if err != nil { diff --git a/runtime/runtime.go b/runtime/runtime.go index ed99b88..14e809c 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -24,6 +24,10 @@ const ( REFERENCE_TYPE_TAG = "tag" ) +const ( + COUNT_FIELD_NAME = "count" +) + var repo *git.Repository var builder *GitBuilder var boolRegister bool diff --git a/runtime/runtime_test.go b/runtime/runtime_test.go index 1e25cfa..c7ad5ca 100644 --- a/runtime/runtime_test.go +++ b/runtime/runtime_test.go @@ -130,3 +130,29 @@ func TestNotFoundCommitWithInStatementAndSorting(t *testing.T) { t.Errorf(errGit.Error()) } } + +func TestFoundCommitsWithSevenInHash(t *testing.T) { + folder, errFile := filepath.Abs("../") + + if errFile != nil { + t.Errorf(errFile.Error()) + } + + query := "select count(*) from commits where '7' in hash order by date desc limit 1000" + + parser.New(query) + ast, errGit := parser.AST() + if errGit != nil { + t.Errorf(errGit.Error()) + } + ast.Path = &folder + errGit = semantical.Analysis(ast) + if errGit != nil { + t.Errorf(errGit.Error()) + } + + typeFormat := "table" + if errGit = Run(ast, &typeFormat); errGit != nil { + t.Errorf(errGit.Error()) + } +} diff --git a/runtime/visitor.go b/runtime/visitor.go index ec8b1f1..1b2701f 100644 --- a/runtime/visitor.go +++ b/runtime/visitor.go @@ -17,7 +17,9 @@ func (v *RuntimeVisitor) VisitSelect(n *parser.NodeSelect) error { proxyTableName := n.Tables[0] // refactor tree proxy := builder.proxyTables[proxyTableName] - if !n.WildCard { + if n.Count { + // do nothing + } else if !n.WildCard { err := testAllFieldsFromTable(n.Fields, proxyTableName) if err != nil { return err @@ -62,7 +64,12 @@ func (v *RuntimeVisitor) VisitSelect(n *parser.NodeSelect) error { if err != nil { return err } - return testAllFieldsFromTable(n.Fields, table) + if n.Count { + n.Fields = []string{COUNT_FIELD_NAME} + }else{ + err = testAllFieldsFromTable(n.Fields, table) + } + return err // Why not visit expression right now ? // Because we will, at first, discover the current object } diff --git a/runtime/visitor_test.go b/runtime/visitor_test.go index 26200a5..c4b2fcc 100644 --- a/runtime/visitor_test.go +++ b/runtime/visitor_test.go @@ -16,6 +16,15 @@ func TestTestAllFieldsInExprBranches(t *testing.T) { } } +func TestTestAllFieldsInExprBranchesWithCount(t *testing.T) { + query := "select count(*) from branches where name = 'something' and somthing > 'name'" + err := parseAndVisitQuery(query, "../", t) + if err == nil { + t.Error("Expected error, received none") + } +} + + func TestTestAllFieldsInExprRefs(t *testing.T) { query := "select * from refs where name = 'something' or type = 'asdfasdfsd'" err := parseAndVisitQuery(query, "../", t) diff --git a/semantical/semantical.go b/semantical/semantical.go index 5f384c0..621257f 100644 --- a/semantical/semantical.go +++ b/semantical/semantical.go @@ -51,7 +51,7 @@ func (v *SemanticalVisitor) VisitSelect(n *parser.NodeSelect) error { } if 0 == n.Limit { - return throwSemanticalError("Limit should be greater then zero") + return throwSemanticalError("Limit should be greater than zero") } return nil diff --git a/semantical/semantical_test.go b/semantical/semantical_test.go index 6584796..f30768d 100644 --- a/semantical/semantical_test.go +++ b/semantical/semantical_test.go @@ -98,7 +98,7 @@ func TestSmallerWithDate(t *testing.T) { } func TestSmallerWithDateWithoutTime(t *testing.T) { - parser.New("select * from commits where date > '2013-03-14'") + parser.New("select count(*) from commits where date > '2013-03-14'") ast, parserErr := parser.AST() if parserErr != nil { t.Fatalf(parserErr.Error())