Merge pull request #3930 from masatake/vim--vim9

Vim:  support vim9script

Units/parser-vim.r/vim-const.d/args.ctags

@@ -0,0 +1,2 @@
+--sort=no
+--fields=+t

Units/parser-vim.r/vim-const.d/expected.tags

@@ -1 +1,5 @@
 s:CONSTANT	input.vim	/^const s:CONSTANT = 42$/;"	C
+matches	input-0.vim	/^final matches = []$/;"	C
+names	input-0.vim	/^const names = ['Betty', 'Peter']$/;"	C
+LINK	input-0.vim	/^const LINK: string = '->'$/;"	C	typeref:typename:string
+mylist	input-0.vim	/^final mylist: list<string> = ['foo']$/;"	C	typeref:typename:list<string>

Units/parser-vim.r/vim-const.d/input-0.vim

@@ -0,0 +1,10 @@
+vim9script
+
+# Taken from https://vim-jp.org/vimdoc-ja/vim9.html#vim9-differences
+final matches = []
+const names = ['Betty', 'Peter']
+
+const LINK: string = '->'
+
+# Taken from ~/var/vim/runtime/doc/vim9.txt
+final mylist: list<string> = ['foo']

Units/parser-vim.r/vim9-def-function.d/args.ctags

@@ -0,0 +1,2 @@
+--sort=no
+--fields=+tS

Units/parser-vim.r/vim9-def-function.d/expected.tags

@@ -1 +1,2 @@
-Func	input.vim	/^def Func()$/;"	f
+Func	input.vim	/^def Func()$/;"	f	signature:()
+Func2	input.vim	/^def Func2(): number$/;"	f	typeref:typename:number	signature:()

Units/parser-vim.r/vim9-def-function.d/input.vim

@@ -1,2 +1,7 @@
 def Func()
 enddef
+
+vim9script
+def Func2(): number
+	return 0
+enddef

Units/parser-vim.r/vim9-export.d/args.ctags

@@ -0,0 +1 @@
+--sort=no

Units/parser-vim.r/vim9-export.d/expected.tags

@@ -0,0 +1,5 @@
+EXPORTED_CONST	input.vim	/^export const EXPORTED_CONST = 1234$/;"	C
+v	input.vim	/^export var v = 1$/;"	v
+f	input.vim	/^export final f = 2$/;"	C
+c	input.vim	/^export const c = 3$/;"	C
+MyFunc	input.vim	/^export def MyFunc()$/;"	f

Units/parser-vim.r/vim9-export.d/input.vim

@@ -0,0 +1,12 @@
+vim9script
+
+export const EXPORTED_CONST = 1234
+export var v = 1
+export final f = 2
+export const c = 3
+export def MyFunc()
+       echo "hello"
+enddef
+
+# export class MyClass ...
+# export interface MyClass ...

Units/parser-vim.r/vim9-var.d/args.ctags

@@ -0,0 +1,2 @@
+--sort=no
+--fields=+St

Units/parser-vim.r/vim9-var.d/expected.tags

@@ -0,0 +1,5 @@
+cur_menu_name	input.vim	/^var cur_menu_name = ""$/;"	v
+cur_menu_nr	input.vim	/^var cur_menu_nr = 0$/;"	v
+cur_menu_item	input.vim	/^var cur_menu_item: number = 0$/;"	v	typeref:typename:number
+cur_menu_char	input.vim	/^var cur_menu_char = ""$/;"	v
+language_section	input.vim	/^var language_section: list<string> =<< trim END$/;"	v	typeref:typename:list<string>

Units/parser-vim.r/vim9-var.d/input.vim

@@ -0,0 +1,17 @@
+vim9script
+
+# Taken from vim/runtime/makemenu.vim
+
+var cur_menu_name = ""
+var cur_menu_nr = 0
+var cur_menu_item: number = 0
+var cur_menu_char = ""
+
+# Taken from vim/runtime/import/dist/vimhighlight.vim
+
+var language_section: list<string> =<< trim END
+
+    Highlighting groups for language syntaxes
+    -----------------------------------------
+
+END

parsers/vim.c

@@ -81,25 +81,16 @@ static kindDefinition VimKinds [] = {
  *  DATA DECLARATIONS
  */
 
-#if 0
-typedef enum eException {
-	ExceptionNone, ExceptionEOF
-} exception_t;
-#endif
-
 /*
  *  DATA DEFINITIONS
  */
-
-#if 0
-static jmp_buf Exception;
-#endif
+static bool vim9script;
 
 /*
  *  FUNCTION DEFINITIONS
  */
 
-static bool parseVimLine (const unsigned char *line, int infunction);
+static bool parseVimLine (const unsigned char *line, int parent);
 
 /* This function takes a char pointer, tries to find a scope separator in the
  * string, and if it does, returns a pointer to the character after the colon,
@@ -227,7 +218,7 @@ static const unsigned char *readVimLine (void)
 		while (isspace (*line))
 			++line;
 
-		if ((int) *line == '"')
+		if ((int) *line == (vim9script? '#': '"'))
 			continue;  /* skip comment */
 
 		break;
@@ -248,9 +239,42 @@ static const unsigned char *readVimballLine (void)
 	return line;
 }
 
-static vString *parseSignature (const unsigned char *cp,
-								tagEntryInfo *e,
-								vString *buf)
+static const unsigned char *parseRettype (const unsigned char *cp, tagEntryInfo *e)
+{
+	while (*cp && isspace (*cp))
+		++cp;
+
+	if (!*cp)
+		return cp;
+
+	vString *buf = vStringNew ();
+	while (*cp && *cp != '#' && *cp != '=')
+	{
+		if (isspace (*cp)
+			&& !vStringIsEmpty(buf)
+			&& isspace (vStringLast(buf)))
+		{
+			++cp;
+			continue;
+		}
+		vStringPut (buf, *cp);
+		++cp;
+	}
+
+	if (vStringIsEmpty(buf))
+		return cp;
+
+	vStringStripTrailing (buf);
+	e->extensionFields.typeRef[0] = eStrdup ("typename");
+	e->extensionFields.typeRef[1] = vStringDeleteUnwrap (buf);
+
+	return cp;
+}
+
+static vString *parseSignatureAndRettype (const unsigned char *cp,
+										  tagEntryInfo *e,
+										  vString *buf,
+										  bool extractRettype)
 {
 	/* TODO capture parameters */
 
@@ -282,12 +306,21 @@ static vString *parseSignature (const unsigned char *cp,
 	{
 		e->extensionFields.signature = vStringDeleteUnwrap (buf);
 		buf = NULL;
+
+		if (extractRettype)
+		{
+			++cp;
+			while (*cp && isspace (*cp))
+				++cp;
+			if (*cp == ':')
+				parseRettype (++cp, e);
+		}
 	}
 
 	return buf;
 }
 
-static void parseFunction (const unsigned char *line)
+static void parseFunction (const unsigned char *line, int parent, bool definedWithDEF)
 {
 	vString *name = vStringNew ();
 	vString *signature = NULL;
@@ -327,12 +360,17 @@ static void parseFunction (const unsigned char *line)
 				vStringClear (name);
 
 				e = getEntryInCorkQueue (index);
-				if (e && isFieldEnabled (FIELD_SIGNATURE))
+				if (e)
 				{
-					while (*cp && isspace (*cp))
-						++cp;
-					if (*cp == '(')
-						signature = parseSignature (cp, e, NULL);
+					e->extensionFields.scopeIndex = parent;
+					if (isFieldEnabled (FIELD_SIGNATURE))
+					{
+						while (*cp && isspace (*cp))
+							++cp;
+						if (*cp == '(')
+							signature = parseSignatureAndRettype (cp, e, NULL,
+																  definedWithDEF);
+					}
 				}
 			}
 		}
@@ -349,7 +387,8 @@ static void parseFunction (const unsigned char *line)
 			/* A backslash at the start of a line stands for a line continuation.
 			 * https://vimhelp.org/repeat.txt.html#line-continuation */
 			if (*cp == '\\')
-				signature = parseSignature (++cp, e, signature);
+				signature = parseSignatureAndRettype (++cp, e, signature,
+													  definedWithDEF);
 		}
 
 		if (wordMatchLen (line, "endfunction", 4) || wordMatchLen (line, "enddef", 6))
@@ -359,7 +398,7 @@ static void parseFunction (const unsigned char *line)
 			break;
 		}
 
-		parseVimLine (line, true);
+		parseVimLine (line, index);
 	}
 	if (signature)
 		vStringDelete (signature);
@@ -519,7 +558,7 @@ static bool parseCommand (const unsigned char *line)
 }
 
 /* =<< trim {endmarker} */
-static int parserHeredocMarker(const unsigned char *cp)
+static int parseHeredocMarker(const unsigned char *cp)
 {
 	while (*cp && isspace (*cp))
 		++cp;
@@ -562,7 +601,7 @@ static int parserHeredocMarker(const unsigned char *cp)
  * If we have a heredoc end marker at the end of LINE,
  * parseVariableOrConstant returns the tag cork index for the marker.
  */
-static int parseVariableOrConstant (const unsigned char *line, int infunction, int kindIndex)
+static int parseVariableOrConstant (const unsigned char *line, int parent, int kindIndex)
 {
 	vString *name = vStringNew ();
 	int heredoc = CORK_NIL;
@@ -594,7 +633,7 @@ static int parseVariableOrConstant (const unsigned char *line, int infunction, i
 			goto cleanUp;
 
 		/* Skip non-global vars in functions */
-		if (infunction && (*np != ':' || *cp != 'g'))
+		if (parent != CORK_NIL && (*np != ':' || *cp != 'g'))
 			goto cleanUp;
 
 		/* deal with spaces, $, @ and & */
@@ -613,10 +652,27 @@ static int parseVariableOrConstant (const unsigned char *line, int infunction, i
 			vStringPut (name, *cp);
 			++cp;
 		} while (isalnum (*cp) || *cp == '_' || *cp == '#' || *cp == ':' || *cp == '$');
-		makeSimpleTag (name, kindIndex);
-		vStringClear (name);
 
-		heredoc = parserHeredocMarker(cp);
+		bool hasType = false;
+		if (*cp && vim9script && !vStringIsEmpty (name) && (vStringLast (name) == ':'))
+		{
+			Assert(line != cp);
+			Assert(*(cp - 1) == ':');
+			vStringChop (name);
+			hasType = true;
+		}
+
+		if (!vStringIsEmpty (name))
+		{
+			int r = makeSimpleTag (name, kindIndex);
+			vStringClear (name);
+
+			tagEntryInfo *e;
+			if (hasType && (e = getEntryInCorkQueue (r)))
+				cp = parseRettype (cp, e);
+
+			heredoc = parseHeredocMarker(cp);
+		}
 	}
 
 cleanUp:
@@ -704,12 +760,25 @@ static bool parseMap (const unsigned char *line)
 	return true;
 }
 
-static bool parseVimLine (const unsigned char *line, int infunction)
+static bool parseVimLine (const unsigned char *line, int parent)
 {
 	bool readNextLine = true;
 	int heredoc = CORK_NIL;
 
-	if (wordMatchLen (line, "command", 3))
+	if (vim9script && wordMatchLen (line, "export", 6))
+	{
+		line += 6;
+		while (*line && isspace (*line))
+			++line;
+		/* TODO: export should be stored to a field. */
+	}
+
+	if (wordMatchLen (line, "vim9script", 10))
+	{
+		vim9script = true;
+	}
+
+	else if (wordMatchLen (line, "command", 3))
 	{
 		readNextLine = parseCommand (line);
 		/* TODO - Handle parseCommand returning false */
@@ -722,21 +791,21 @@ static bool parseVimLine (const unsigned char *line, int infunction)
 
 	else if (wordMatchLen (line, "function", 2) || wordMatchLen (line, "def", 3))
 	{
-		parseFunction (skipWord (line));
+		parseFunction (skipWord (line), parent, vim9script && line[0] == 'd');
 	}
 
 	else if (wordMatchLen (line, "augroup", 3))
 	{
 		parseAutogroup (skipWord (line));
 	}
 
-	else if (wordMatchLen (line, "let", 3))
+	else if (wordMatchLen (line, "let", 3) || (vim9script && wordMatchLen (line, "var", 3)))
 	{
-		heredoc = parseVariableOrConstant (skipWord (line), infunction, K_VARIABLE);
+		heredoc = parseVariableOrConstant (skipWord (line), parent, K_VARIABLE);
 	}
-	else if (wordMatchLen (line, "const", 4))
+	else if (wordMatchLen (line, "const", 4) || wordMatchLen (line, "final", 5))
 	{
-		heredoc = parseVariableOrConstant (skipWord (line), infunction, K_CONST);
+		heredoc = parseVariableOrConstant (skipWord (line), parent, K_CONST);
 	}
 
 	tagEntryInfo *e;
@@ -760,13 +829,13 @@ static bool parseVimLine (const unsigned char *line, int infunction)
 	return readNextLine;
 }
 
-static void parseVimFile (const unsigned char *line)
+static void parseVimFile (const unsigned char *line, int parent)
 {
 	bool readNextLine = true;
 
 	while (line != NULL)
 	{
-		readNextLine = parseVimLine (line, false);
+		readNextLine = parseVimLine (line, parent);
 
 		if (readNextLine)
 			line = readVimLine ();
@@ -850,6 +919,8 @@ static void findVimTags (void)
 	const unsigned char *line;
 	/* TODO - change this into a structure */
 
+	vim9script = false;
+
 	line = readVimLine ();
 
 	if (line == NULL)
@@ -863,7 +934,7 @@ static void findVimTags (void)
 	}
 	else
 	{
-		parseVimFile (line);
+		parseVimFile (line, CORK_NIL);
 	}
 }