internal comments (#268)

* internal comments

* ignore comments starting with bang

---------

Co-authored-by: lukasjenicek <lukas.jenicek5@gmail.com>

schema/ridl/_example/comments.ridl

@@ -10,7 +10,7 @@ enum Kind: uint32
   - USER = 1             # user
   - ADMIN = 2            # comment..
 
-# or.. just..
+#! or.. just..
 enum Kind2: uint32
   - USER                 # aka, = 0
   - ADMIN                # aka, = 1

schema/ridl/lexer.go

@@ -63,6 +63,7 @@ const (
 	tokenDot                         // "."
 	tokenQuestionMark                // "?"
 	tokenRocket                      // "=>"
+	tokenBang                        // "!"
 	tokenWord                        // ..wordCharset..
 
 	tokenExtra // other
@@ -97,6 +98,7 @@ var tokenTypeName = map[tokenType]string{
 	tokenSlash:             "[slash]",
 	tokenQuestionMark:      "[question mark]",
 	tokenRocket:            "[rocket]",
+	tokenBang:              "[bang]",
 	tokenWord:              "[word]",
 	tokenExtra:             "[extra]",
 	tokenComposed:          "[composed]",
@@ -123,6 +125,7 @@ var tokenTypeValue = map[tokenType][]rune{
 	tokenComma:             {','},
 	tokenDot:               {'.'},
 	tokenQuestionMark:      {'?'},
+	tokenBang:              {'!'},
 }
 
 var (
@@ -145,6 +148,7 @@ var (
 	isBackslash         = isTokenType(tokenBackslash)
 	isSlash             = isTokenType(tokenSlash)
 	isDot               = isTokenType(tokenDot)
+	isBang              = isTokenType(tokenBang)
 )
 
 func isTokenType(tt tokenType) func(r rune) bool {

schema/ridl/parser.go

@@ -523,7 +523,9 @@ func parseComments(comments map[int]string, currentLine int) string {
 	for ; currentLine >= 0; currentLine-- {
 		comment, ok := comments[currentLine]
 		if ok {
-			c = append(c, comment)
+			if !strings.HasPrefix(comment, "!") {
+				c = append(c, comment)
+			}
 			delete(comments, currentLine)
 			iteration = 0
 

schema/ridl/parser_test.go

@@ -994,9 +994,14 @@ func TestParseServiceComments(t *testing.T) {
 		# Contacts service line 1
 		# Contacts service line 2
 		service ContactsService  # Contacts service line 3
+			#! skip this line
 			# GetContact gives you contact for specific id
 			- GetContact(id: int) => (contact: Contact)
 			# Version returns you current deployed version
+			# 
+			#! skip this line as its internal comment
+			#! skip more lines
+			#! more
 			- Version() => (details: any)
 		`)
 	assert.NoError(t, err)
@@ -1011,5 +1016,5 @@ func TestParseServiceComments(t *testing.T) {
 
 	assert.Equal(t, "Contacts service line 1\nContacts service line 2\nContacts service line 3", serviceNode.comment)
 	assert.Equal(t, "GetContact gives you contact for specific id", serviceNode.methods[0].comment)
-	assert.Equal(t, "Version returns you current deployed version", serviceNode.methods[1].comment)
+	assert.Equal(t, "Version returns you current deployed version\n", serviceNode.methods[1].comment)
 }

schema/ridl/tokenizer.go

@@ -16,6 +16,7 @@ func tokenize(src []byte) ([]token, map[int]string, error) {
 
 	commentLine := false
 	commentTokens := []string{}
+
 	for tok := range lx.tokens {
 		if tok.tt == tokenEOF {
 			break