From a77d85abdc3779a97b9c427399b156f142931543 Mon Sep 17 00:00:00 2001 From: David Yu Yang Date: Sat, 20 Nov 2021 22:39:42 +0800 Subject: [PATCH] #41 Add GDScript support Heavily borrowed from the python lexer because GDScript is heavily inspired by python. --- cppcheck.suppress | 2 + doc/LexillaHistory.html | 6 + include/LexicalStyles.iface | 3 +- include/SciLexer.h | 1 + lexers/LexGDScript.cxx | 735 ++++++++++++++++++ src/Lexilla.cxx | 2 + src/Lexilla/Lexilla.xcodeproj/project.pbxproj | 4 + src/deps.mak | 17 + src/lexilla.mak | 1 + src/nmdeps.mak | 17 + test/examples/gdscript/AllStyles.gd | 51 ++ test/examples/gdscript/AllStyles.gd.folded | 52 ++ test/examples/gdscript/AllStyles.gd.styled | 51 ++ test/examples/gdscript/SciTE.properties | 5 + 14 files changed, 946 insertions(+), 1 deletion(-) create mode 100644 lexers/LexGDScript.cxx create mode 100644 test/examples/gdscript/AllStyles.gd create mode 100644 test/examples/gdscript/AllStyles.gd.folded create mode 100644 test/examples/gdscript/AllStyles.gd.styled create mode 100644 test/examples/gdscript/SciTE.properties diff --git a/cppcheck.suppress b/cppcheck.suppress index ef04099d..9e7645a3 100644 --- a/cppcheck.suppress +++ b/cppcheck.suppress @@ -41,6 +41,7 @@ knownConditionTrueFalse:lexilla/lexers/LexEScript.cxx constParameter:lexilla/lexers/LexFortran.cxx redundantCondition:lexilla/lexers/LexFSharp.cxx knownConditionTrueFalse:lexilla/lexers/LexFSharp.cxx +constParameter:lexilla/lexers/LexGDScript.cxx variableScope:lexilla/lexers/LexGui4Cli.cxx constParameter:lexilla/lexers/LexHaskell.cxx constParameter:lexilla/lexers/LexHex.cxx @@ -105,6 +106,7 @@ knownConditionTrueFalse:lexilla/lexers/LexYAML.cxx // These are due to Accessor::IndentAmount not declaring the callback as taking a const. // Changing this could cause problems for downstream projects. constParameterCallback:lexilla/lexers/LexEiffel.cxx +constParameterCallback:lexilla/lexers/LexGDScript.cxx constParameterCallback:lexilla/lexers/LexPython.cxx constParameterCallback:lexilla/lexers/LexScriptol.cxx constParameterCallback:lexilla/lexers/LexVB.cxx diff --git a/doc/LexillaHistory.html b/doc/LexillaHistory.html index 06a7a016..369e5cc5 100644 --- a/doc/LexillaHistory.html +++ b/doc/LexillaHistory.html @@ -573,6 +573,7 @@

Contributors

Michael Heath Antonio Cebrián + David Yu Yang

Releases

@@ -588,6 +589,11 @@

Pull request #39.
  • + Add provisional GDScript lexer. + Some behaviour and lexical states may change before this lexer is stable. + Pull request #41. +
    • +
  • Fix strings ending in escaped '\' in F#. Issue #38,
    • diff --git a/include/LexicalStyles.iface b/include/LexicalStyles.iface index 413997dd..a5108744 100644 --- a/include/LexicalStyles.iface +++ b/include/LexicalStyles.iface @@ -143,6 +143,7 @@ val SCLEX_RAKU=131 val SCLEX_FSHARP=132 val SCLEX_JULIA=133 val SCLEX_ASCIIDOC=134 +val SCLEX_GDSCRIPT=135 # When a lexer specifies its language as SCLEX_AUTOMATIC it receives a # value assigned in sequence from SCLEX_AUTOMATIC+1. @@ -150,6 +151,7 @@ val SCLEX_AUTOMATIC=1000 # Lexical states for SCLEX_PYTHON lex Python=SCLEX_PYTHON SCE_P_ lex Nimrod=SCLEX_NIMROD SCE_P_ +lex GDScript=SCLEX_GDSCRIPT SCE_P_ val SCE_P_DEFAULT=0 val SCE_P_COMMENTLINE=1 val SCE_P_NUMBER=2 @@ -2277,4 +2279,3 @@ val SCE_ASCIIDOC_LITERALBK=20 val SCE_ASCIIDOC_ATTRIB=21 val SCE_ASCIIDOC_ATTRIBVAL=22 val SCE_ASCIIDOC_MACRO=23 - diff --git a/include/SciLexer.h b/include/SciLexer.h index 2fc1192b..952519b6 100644 --- a/include/SciLexer.h +++ b/include/SciLexer.h @@ -147,6 +147,7 @@ #define SCLEX_FSHARP 132 #define SCLEX_JULIA 133 #define SCLEX_ASCIIDOC 134 +#define SCLEX_GDSCRIPT 135 #define SCLEX_AUTOMATIC 1000 #define SCE_P_DEFAULT 0 #define SCE_P_COMMENTLINE 1 diff --git a/lexers/LexGDScript.cxx b/lexers/LexGDScript.cxx new file mode 100644 index 00000000..6e4525eb --- /dev/null +++ b/lexers/LexGDScript.cxx @@ -0,0 +1,735 @@ +// Scintilla source code edit control +/** @file LexGDScript.cxx + ** Lexer for GDScript. + **/ +// Copyright 1998-2002 by Neil Hodgson +// Heavily modified later for GDScript +// The License.txt file describes the conditions under which this software may be distributed. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "ILexer.h" +#include "Scintilla.h" +#include "SciLexer.h" + +#include "StringCopy.h" +#include "WordList.h" +#include "LexAccessor.h" +#include "Accessor.h" +#include "StyleContext.h" +#include "CharacterSet.h" +#include "CharacterCategory.h" +#include "LexerModule.h" +#include "OptionSet.h" +#include "SubStyles.h" +#include "DefaultLexer.h" + +using namespace Scintilla; +using namespace Lexilla; + +namespace { + +enum kwType { kwOther, kwClass, kwDef, kwExtends}; + +constexpr int indicatorWhitespace = 1; + +bool IsGDStringStart(int ch) { + return (ch == '\'' || ch == '"'); +} + +bool IsGDComment(Accessor &styler, Sci_Position pos, Sci_Position len) { + return len > 0 && styler[pos] == '#'; +} + +constexpr bool IsGDSingleQuoteStringState(int st) noexcept { + return ((st == SCE_P_CHARACTER) || (st == SCE_P_STRING) || + (st == SCE_P_FCHARACTER) || (st == SCE_P_FSTRING)); +} + +constexpr bool IsGDTripleQuoteStringState(int st) noexcept { + return ((st == SCE_P_TRIPLE) || (st == SCE_P_TRIPLEDOUBLE) || + (st == SCE_P_FTRIPLE) || (st == SCE_P_FTRIPLEDOUBLE)); +} + +char GetGDStringQuoteChar(int st) noexcept { + if ((st == SCE_P_CHARACTER) || (st == SCE_P_TRIPLE)) + return '\''; + if ((st == SCE_P_STRING) || (st == SCE_P_TRIPLEDOUBLE)) + return '"'; + + return '\0'; +} + +/* Return the state to use for the string starting at i; *nextIndex will be set to the first index following the quote(s) */ +int GetGDStringState(Accessor &styler, Sci_Position i, Sci_PositionU *nextIndex) { + char ch = styler.SafeGetCharAt(i); + char chNext = styler.SafeGetCharAt(i + 1); + + if (ch != '"' && ch != '\'') { + *nextIndex = i + 1; + return SCE_P_DEFAULT; + } + + if (ch == chNext && ch == styler.SafeGetCharAt(i + 2)) { + *nextIndex = i + 3; + + if (ch == '"') + return SCE_P_TRIPLEDOUBLE; + else + return SCE_P_TRIPLE; + } else { + *nextIndex = i + 1; + + if (ch == '"') + return SCE_P_STRING; + else + return SCE_P_CHARACTER; + } +} + +inline bool IsAWordChar(int ch, bool unicodeIdentifiers) { + if (IsASCII(ch)) + return (IsAlphaNumeric(ch) || ch == '.' || ch == '_'); + + if (!unicodeIdentifiers) + return false; + + return IsXidContinue(ch); +} + +inline bool IsAWordStart(int ch, bool unicodeIdentifiers) { + if (IsASCII(ch)) + return (IsUpperOrLowerCase(ch) || ch == '_'); + + if (!unicodeIdentifiers) + return false; + + return IsXidStart(ch); +} + +bool IsFirstNonWhitespace(Sci_Position pos, Accessor &styler) { + const Sci_Position line = styler.GetLine(pos); + const Sci_Position start_pos = styler.LineStart(line); + for (Sci_Position i = start_pos; i < pos; i++) { + const char ch = styler[i]; + if (!(ch == ' ' || ch == '\t')) + return false; + } + return true; +} + +// Options used for LexerGDScript +struct OptionsGDScript { + int whingeLevel; + bool base2or8Literals; + bool stringsOverNewline; + bool keywords2NoSubIdentifiers; + bool fold; + bool foldQuotes; + bool foldCompact; + bool unicodeIdentifiers; + + OptionsGDScript() noexcept { + whingeLevel = 0; + base2or8Literals = true; + stringsOverNewline = false; + keywords2NoSubIdentifiers = false; + fold = false; + foldQuotes = false; + foldCompact = false; + unicodeIdentifiers = true; + } +}; + +const char *const gdscriptWordListDesc[] = { + "Keywords", + "Highlighted identifiers", + nullptr +}; + +struct OptionSetGDScript : public OptionSet { + OptionSetGDScript() { + DefineProperty("lexer.gdscript.whinge.level", &OptionsGDScript::whingeLevel, + "For GDScript code, checks whether indenting is consistent. " + "The default, 0 turns off indentation checking, " + "1 checks whether each line is potentially inconsistent with the previous line, " + "2 checks whether any space characters occur before a tab character in the indentation, " + "3 checks whether any spaces are in the indentation, and " + "4 checks for any tab characters in the indentation. " + "1 is a good level to use."); + + DefineProperty("lexer.gdscript.literals.binary", &OptionsGDScript::base2or8Literals, + "Set to 0 to not recognise binary and octal literals: 0b1011 0o712."); + + DefineProperty("lexer.gdscript.strings.over.newline", &OptionsGDScript::stringsOverNewline, + "Set to 1 to allow strings to span newline characters."); + + DefineProperty("lexer.gdscript.keywords2.no.sub.identifiers", &OptionsGDScript::keywords2NoSubIdentifiers, + "When enabled, it will not style keywords2 items that are used as a sub-identifier. " + "Example: when set, will not highlight \"foo.open\" when \"open\" is a keywords2 item."); + + DefineProperty("fold", &OptionsGDScript::fold); + + DefineProperty("fold.gdscript.quotes", &OptionsGDScript::foldQuotes, + "This option enables folding multi-line quoted strings when using the GDScript lexer."); + + DefineProperty("fold.compact", &OptionsGDScript::foldCompact); + + DefineProperty("lexer.gdscript.unicode.identifiers", &OptionsGDScript::unicodeIdentifiers, + "Set to 0 to not recognise Unicode identifiers."); + + DefineWordListSets(gdscriptWordListDesc); + } +}; + +const char styleSubable[] = { SCE_P_IDENTIFIER, 0 }; + +LexicalClass lexicalClasses[] = { + // Lexer GDScript SCLEX_GDSCRIPT SCE_P_: + 0, "SCE_P_DEFAULT", "default", "White space", + 1, "SCE_P_COMMENTLINE", "comment line", "Comment", + 2, "SCE_P_NUMBER", "literal numeric", "Number", + 3, "SCE_P_STRING", "literal string", "String", + 4, "SCE_P_CHARACTER", "literal string", "Single quoted string", + 5, "SCE_P_WORD", "keyword", "Keyword", + 6, "SCE_P_TRIPLE", "literal string", "Triple quotes", + 7, "SCE_P_TRIPLEDOUBLE", "literal string", "Triple double quotes", + 8, "SCE_P_CLASSNAME", "identifier", "Class name definition", + 9, "SCE_P_DEFNAME", "identifier", "Function or method name definition", + 10, "SCE_P_OPERATOR", "operator", "Operators", + 11, "SCE_P_IDENTIFIER", "identifier", "Identifiers", + 12, "SCE_P_COMMENTBLOCK", "comment", "Comment-blocks", + 13, "SCE_P_STRINGEOL", "error literal string", "End of line where string is not closed", + 14, "SCE_P_WORD2", "identifier", "Highlighted identifiers", + 15, "SCE_P_DECORATOR", "preprocessor", "Decorators", +}; + +} + +class LexerGDScript : public DefaultLexer { + WordList keywords; + WordList keywords2; + OptionsGDScript options; + OptionSetGDScript osGDScript; + enum { ssIdentifier }; + SubStyles subStyles; +public: + explicit LexerGDScript() : + DefaultLexer("gdscript", SCLEX_GDSCRIPT, lexicalClasses, ELEMENTS(lexicalClasses)), + subStyles(styleSubable, 0x80, 0x40, 0) { + } + ~LexerGDScript() override { + } + void SCI_METHOD Release() override { + delete this; + } + int SCI_METHOD Version() const override { + return lvRelease5; + } + const char *SCI_METHOD PropertyNames() override { + return osGDScript.PropertyNames(); + } + int SCI_METHOD PropertyType(const char *name) override { + return osGDScript.PropertyType(name); + } + const char *SCI_METHOD DescribeProperty(const char *name) override { + return osGDScript.DescribeProperty(name); + } + Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override; + const char * SCI_METHOD PropertyGet(const char *key) override { + return osGDScript.PropertyGet(key); + } + const char *SCI_METHOD DescribeWordListSets() override { + return osGDScript.DescribeWordListSets(); + } + Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override; + void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override; + void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override; + + void *SCI_METHOD PrivateCall(int, void *) override { + return nullptr; + } + + int SCI_METHOD LineEndTypesSupported() override { + return SC_LINE_END_TYPE_UNICODE; + } + + int SCI_METHOD AllocateSubStyles(int styleBase, int numberStyles) override { + return subStyles.Allocate(styleBase, numberStyles); + } + int SCI_METHOD SubStylesStart(int styleBase) override { + return subStyles.Start(styleBase); + } + int SCI_METHOD SubStylesLength(int styleBase) override { + return subStyles.Length(styleBase); + } + int SCI_METHOD StyleFromSubStyle(int subStyle) override { + const int styleBase = subStyles.BaseStyle(subStyle); + return styleBase; + } + int SCI_METHOD PrimaryStyleFromStyle(int style) override { + return style; + } + void SCI_METHOD FreeSubStyles() override { + subStyles.Free(); + } + void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override { + subStyles.SetIdentifiers(style, identifiers); + } + int SCI_METHOD DistanceToSecondaryStyles() override { + return 0; + } + const char *SCI_METHOD GetSubStyleBases() override { + return styleSubable; + } + + static ILexer5 *LexerFactoryGDScript() { + return new LexerGDScript(); + } + +private: + void ProcessLineEnd(StyleContext &sc, bool &inContinuedString); +}; + +Sci_Position SCI_METHOD LexerGDScript::PropertySet(const char *key, const char *val) { + if (osGDScript.PropertySet(&options, key, val)) { + return 0; + } + return -1; +} + +Sci_Position SCI_METHOD LexerGDScript::WordListSet(int n, const char *wl) { + WordList *wordListN = nullptr; + switch (n) { + case 0: + wordListN = &keywords; + break; + case 1: + wordListN = &keywords2; + break; + default: + break; + } + Sci_Position firstModification = -1; + if (wordListN) { + WordList wlNew; + wlNew.Set(wl); + if (*wordListN != wlNew) { + wordListN->Set(wl); + firstModification = 0; + } + } + return firstModification; +} + +void LexerGDScript::ProcessLineEnd(StyleContext &sc, bool &inContinuedString) { + if ((sc.state == SCE_P_DEFAULT) + || IsGDTripleQuoteStringState(sc.state)) { + // Perform colourisation of white space and triple quoted strings at end of each line to allow + // tab marking to work inside white space and triple quoted strings + sc.SetState(sc.state); + } + + if (IsGDSingleQuoteStringState(sc.state)) { + if (inContinuedString || options.stringsOverNewline) { + inContinuedString = false; + } else { + sc.ChangeState(SCE_P_STRINGEOL); + sc.ForwardSetState(SCE_P_DEFAULT); + } + } +} + +void SCI_METHOD LexerGDScript::Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) { + Accessor styler(pAccess, nullptr); + + const Sci_Position endPos = startPos + length; + + // Backtrack to previous line in case need to fix its tab whinging + Sci_Position lineCurrent = styler.GetLine(startPos); + if (startPos > 0) { + if (lineCurrent > 0) { + lineCurrent--; + // Look for backslash-continued lines + while (lineCurrent > 0) { + const Sci_Position eolPos = styler.LineStart(lineCurrent) - 1; + const int eolStyle = styler.StyleAt(eolPos); + if (eolStyle == SCE_P_STRING || eolStyle == SCE_P_CHARACTER + || eolStyle == SCE_P_STRINGEOL) { + lineCurrent -= 1; + } else { + break; + } + } + startPos = styler.LineStart(lineCurrent); + } + initStyle = startPos == 0 ? SCE_P_DEFAULT : styler.StyleAt(startPos - 1); + } + + initStyle = initStyle & 31; + if (initStyle == SCE_P_STRINGEOL) { + initStyle = SCE_P_DEFAULT; + } + + kwType kwLast = kwOther; + int spaceFlags = 0; + styler.IndentAmount(lineCurrent, &spaceFlags, IsGDComment); + bool base_n_number = false; + + const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_P_IDENTIFIER); + + StyleContext sc(startPos, endPos - startPos, initStyle, styler); + + bool indentGood = true; + Sci_Position startIndicator = sc.currentPos; + bool inContinuedString = false; + + for (; sc.More(); sc.Forward()) { + + if (sc.atLineStart) { + styler.IndentAmount(lineCurrent, &spaceFlags, IsGDComment); + indentGood = true; + if (options.whingeLevel == 1) { + indentGood = (spaceFlags & wsInconsistent) == 0; + } else if (options.whingeLevel == 2) { + indentGood = (spaceFlags & wsSpaceTab) == 0; + } else if (options.whingeLevel == 3) { + indentGood = (spaceFlags & wsSpace) == 0; + } else if (options.whingeLevel == 4) { + indentGood = (spaceFlags & wsTab) == 0; + } + if (!indentGood) { + styler.IndicatorFill(startIndicator, sc.currentPos, indicatorWhitespace, 0); + startIndicator = sc.currentPos; + } + } + + if (sc.atLineEnd) { + ProcessLineEnd(sc, inContinuedString); + lineCurrent++; + if (!sc.More()) + break; + } + + bool needEOLCheck = false; + + if (sc.state == SCE_P_OPERATOR) { + kwLast = kwOther; + sc.SetState(SCE_P_DEFAULT); + } else if (sc.state == SCE_P_NUMBER) { + if (!IsAWordChar(sc.ch, false) && + !(!base_n_number && ((sc.ch == '+' || sc.ch == '-') && (sc.chPrev == 'e' || sc.chPrev == 'E')))) { + sc.SetState(SCE_P_DEFAULT); + } + } else if (sc.state == SCE_P_IDENTIFIER) { + if ((sc.ch == '.') || (!IsAWordChar(sc.ch, options.unicodeIdentifiers))) { + char s[100]; + sc.GetCurrent(s, sizeof(s)); + int style = SCE_P_IDENTIFIER; + if (keywords.InList(s)) { + style = SCE_P_WORD; + } else if (kwLast == kwClass) { + style = SCE_P_CLASSNAME; + } else if (kwLast == kwDef) { + style = SCE_P_DEFNAME; + } else if (keywords2.InList(s)) { + if (options.keywords2NoSubIdentifiers) { + // We don't want to highlight keywords2 + // that are used as a sub-identifier, + // i.e. not open in "foo.open". + const Sci_Position pos = styler.GetStartSegment() - 1; + if (pos < 0 || (styler.SafeGetCharAt(pos, '\0') != '.')) + style = SCE_P_WORD2; + } else { + style = SCE_P_WORD2; + } + } else { + const int subStyle = classifierIdentifiers.ValueFor(s); + if (subStyle >= 0) { + style = subStyle; + } + } + sc.ChangeState(style); + sc.SetState(SCE_P_DEFAULT); + if (style == SCE_P_WORD) { + if (0 == strcmp(s, "class")) + kwLast = kwClass; + else if (0 == strcmp(s, "func")) + kwLast = kwDef; + else if (0 == strcmp(s, "extends")) + kwLast = kwExtends; + else + kwLast = kwOther; + } else { + kwLast = kwOther; + } + } + } else if ((sc.state == SCE_P_COMMENTLINE) || (sc.state == SCE_P_COMMENTBLOCK)) { + if (sc.ch == '\r' || sc.ch == '\n') { + sc.SetState(SCE_P_DEFAULT); + } + } else if (sc.state == SCE_P_DECORATOR) { + if (!IsAWordStart(sc.ch, options.unicodeIdentifiers)) { + sc.SetState(SCE_P_DEFAULT); + } + } else if (IsGDSingleQuoteStringState(sc.state)) { + if (sc.ch == '\\') { + if ((sc.chNext == '\r') && (sc.GetRelative(2) == '\n')) { + sc.Forward(); + } + if (sc.chNext == '\n' || sc.chNext == '\r') { + inContinuedString = true; + } else { + // Don't roll over the newline. + sc.Forward(); + } + } else if (sc.ch == GetGDStringQuoteChar(sc.state)) { + sc.ForwardSetState(SCE_P_DEFAULT); + needEOLCheck = true; + } + } else if (sc.state == SCE_P_TRIPLE) { + if (sc.ch == '\\') { + sc.Forward(); + } else if (sc.Match(R"(''')")) { + sc.Forward(); + sc.Forward(); + sc.ForwardSetState(SCE_P_DEFAULT); + needEOLCheck = true; + } + } else if (sc.state == SCE_P_TRIPLEDOUBLE) { + if (sc.ch == '\\') { + sc.Forward(); + } else if (sc.Match(R"(""")")) { + sc.Forward(); + sc.Forward(); + sc.ForwardSetState(SCE_P_DEFAULT); + needEOLCheck = true; + } + } + + if (!indentGood && !IsASpaceOrTab(sc.ch)) { + styler.IndicatorFill(startIndicator, sc.currentPos, indicatorWhitespace, 1); + startIndicator = sc.currentPos; + indentGood = true; + } + + // State exit code may have moved on to end of line + if (needEOLCheck && sc.atLineEnd) { + ProcessLineEnd(sc, inContinuedString); + lineCurrent++; + styler.IndentAmount(lineCurrent, &spaceFlags, IsGDComment); + if (!sc.More()) + break; + } + + // Check for a new state starting character + if (sc.state == SCE_P_DEFAULT) { + if (IsADigit(sc.ch) || (sc.ch == '.' && IsADigit(sc.chNext))) { + if (sc.ch == '0' && (sc.chNext == 'x' || sc.chNext == 'X')) { + base_n_number = true; + sc.SetState(SCE_P_NUMBER); + } else if (sc.ch == '0' && + (sc.chNext == 'o' || sc.chNext == 'O' || sc.chNext == 'b' || sc.chNext == 'B')) { + if (options.base2or8Literals) { + base_n_number = true; + sc.SetState(SCE_P_NUMBER); + } else { + sc.SetState(SCE_P_NUMBER); + sc.ForwardSetState(SCE_P_IDENTIFIER); + } + } else { + base_n_number = false; + sc.SetState(SCE_P_NUMBER); + } + } else if (isoperator(sc.ch) || sc.ch == '`') { + sc.SetState(SCE_P_OPERATOR); + } else if (sc.ch == '#') { + sc.SetState(sc.chNext == '#' ? SCE_P_COMMENTBLOCK : SCE_P_COMMENTLINE); + } else if (sc.ch == '@') { + if (IsFirstNonWhitespace(sc.currentPos, styler)) + sc.SetState(SCE_P_DECORATOR); + else + sc.SetState(SCE_P_OPERATOR); + } else if (IsGDStringStart(sc.ch)) { + Sci_PositionU nextIndex = 0; + sc.SetState(GetGDStringState(styler, sc.currentPos, &nextIndex)); + while (nextIndex > (sc.currentPos + 1) && sc.More()) { + sc.Forward(); + } + } else if (IsAWordStart(sc.ch, options.unicodeIdentifiers)) { + sc.SetState(SCE_P_IDENTIFIER); + } + } + } + styler.IndicatorFill(startIndicator, sc.currentPos, indicatorWhitespace, 0); + sc.Complete(); +} + +static bool IsCommentLine(Sci_Position line, Accessor &styler) { + const Sci_Position pos = styler.LineStart(line); + const Sci_Position eol_pos = styler.LineStart(line + 1) - 1; + for (Sci_Position i = pos; i < eol_pos; i++) { + const char ch = styler[i]; + if (ch == '#') + return true; + else if (ch != ' ' && ch != '\t') + return false; + } + return false; +} + +static bool IsQuoteLine(Sci_Position line, const Accessor &styler) { + const int style = styler.StyleAt(styler.LineStart(line)) & 31; + return IsGDTripleQuoteStringState(style); +} + + +void SCI_METHOD LexerGDScript::Fold(Sci_PositionU startPos, Sci_Position length, int /*initStyle - unused*/, IDocument *pAccess) { + if (!options.fold) + return; + + Accessor styler(pAccess, nullptr); + + const Sci_Position maxPos = startPos + length; + const Sci_Position maxLines = (maxPos == styler.Length()) ? styler.GetLine(maxPos) : styler.GetLine(maxPos - 1); // Requested last line + const Sci_Position docLines = styler.GetLine(styler.Length()); // Available last line + + // Backtrack to previous non-blank line so we can determine indent level + // for any white space lines (needed esp. within triple quoted strings) + // and so we can fix any preceding fold level (which is why we go back + // at least one line in all cases) + int spaceFlags = 0; + Sci_Position lineCurrent = styler.GetLine(startPos); + int indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, nullptr); + while (lineCurrent > 0) { + lineCurrent--; + indentCurrent = styler.IndentAmount(lineCurrent, &spaceFlags, nullptr); + if (!(indentCurrent & SC_FOLDLEVELWHITEFLAG) && + (!IsCommentLine(lineCurrent, styler)) && + (!IsQuoteLine(lineCurrent, styler))) + break; + } + int indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK; + + // Set up initial loop state + startPos = styler.LineStart(lineCurrent); + int prev_state = SCE_P_DEFAULT & 31; + if (lineCurrent >= 1) + prev_state = styler.StyleAt(startPos - 1) & 31; + int prevQuote = options.foldQuotes && IsGDTripleQuoteStringState(prev_state); + + // Process all characters to end of requested range or end of any triple quote + //that hangs over the end of the range. Cap processing in all cases + // to end of document (in case of unclosed quote at end). + while ((lineCurrent <= docLines) && ((lineCurrent <= maxLines) || prevQuote)) { + + // Gather info + int lev = indentCurrent; + Sci_Position lineNext = lineCurrent + 1; + int indentNext = indentCurrent; + int quote = false; + if (lineNext <= docLines) { + // Information about next line is only available if not at end of document + indentNext = styler.IndentAmount(lineNext, &spaceFlags, nullptr); + const Sci_Position lookAtPos = (styler.LineStart(lineNext) == styler.Length()) ? styler.Length() - 1 : styler.LineStart(lineNext); + const int style = styler.StyleAt(lookAtPos) & 31; + quote = options.foldQuotes && IsGDTripleQuoteStringState(style); + } + const bool quote_start = (quote && !prevQuote); + const bool quote_continue = (quote && prevQuote); + if (!quote || !prevQuote) + indentCurrentLevel = indentCurrent & SC_FOLDLEVELNUMBERMASK; + if (quote) + indentNext = indentCurrentLevel; + if (indentNext & SC_FOLDLEVELWHITEFLAG) + indentNext = SC_FOLDLEVELWHITEFLAG | indentCurrentLevel; + + if (quote_start) { + // Place fold point at start of triple quoted string + lev |= SC_FOLDLEVELHEADERFLAG; + } else if (quote_continue || prevQuote) { + // Add level to rest of lines in the string + lev = lev + 1; + } + + // Skip past any blank lines for next indent level info; we skip also + // comments (all comments, not just those starting in column 0) + // which effectively folds them into surrounding code rather + // than screwing up folding. If comments end file, use the min + // comment indent as the level after + + int minCommentLevel = indentCurrentLevel; + while (!quote && + (lineNext < docLines) && + ((indentNext & SC_FOLDLEVELWHITEFLAG) || (IsCommentLine(lineNext, styler)))) { + + if (IsCommentLine(lineNext, styler) && indentNext < minCommentLevel) { + minCommentLevel = indentNext; + } + + lineNext++; + indentNext = styler.IndentAmount(lineNext, &spaceFlags, nullptr); + } + + const int levelAfterComments = ((lineNext < docLines) ? indentNext & SC_FOLDLEVELNUMBERMASK : minCommentLevel); + const int levelBeforeComments = std::max(indentCurrentLevel, levelAfterComments); + + // Now set all the indent levels on the lines we skipped + // Do this from end to start. Once we encounter one line + // which is indented more than the line after the end of + // the comment-block, use the level of the block before + + Sci_Position skipLine = lineNext; + int skipLevel = levelAfterComments; + + while (--skipLine > lineCurrent) { + const int skipLineIndent = styler.IndentAmount(skipLine, &spaceFlags, nullptr); + + if (options.foldCompact) { + if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments) + skipLevel = levelBeforeComments; + + const int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG; + + styler.SetLevel(skipLine, skipLevel | whiteFlag); + } else { + if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > levelAfterComments && + !(skipLineIndent & SC_FOLDLEVELWHITEFLAG) && + !IsCommentLine(skipLine, styler)) + skipLevel = levelBeforeComments; + + styler.SetLevel(skipLine, skipLevel); + } + } + + // Set fold header on non-quote line + if (!quote && !(indentCurrent & SC_FOLDLEVELWHITEFLAG)) { + if ((indentCurrent & SC_FOLDLEVELNUMBERMASK) < (indentNext & SC_FOLDLEVELNUMBERMASK)) + lev |= SC_FOLDLEVELHEADERFLAG; + } + + // Keep track of triple quote state of previous line + prevQuote = quote; + + // Set fold level for this line and move to next line + styler.SetLevel(lineCurrent, options.foldCompact ? lev : lev & ~SC_FOLDLEVELWHITEFLAG); + indentCurrent = indentNext; + lineCurrent = lineNext; + } + + // NOTE: Cannot set level of last line here because indentCurrent doesn't have + // header flag set; the loop above is crafted to take care of this case! + //styler.SetLevel(lineCurrent, indentCurrent); +} + +LexerModule lmGDScript(SCLEX_GDSCRIPT, LexerGDScript::LexerFactoryGDScript, "gdscript", + gdscriptWordListDesc); diff --git a/src/Lexilla.cxx b/src/Lexilla.cxx index c174db15..71400141 100644 --- a/src/Lexilla.cxx +++ b/src/Lexilla.cxx @@ -77,6 +77,7 @@ extern LexerModule lmFortran; extern LexerModule lmFreeBasic; extern LexerModule lmFSharp; extern LexerModule lmGAP; +extern LexerModule lmGDScript; extern LexerModule lmGui4Cli; extern LexerModule lmHaskell; extern LexerModule lmHollywood; @@ -224,6 +225,7 @@ void AddEachLexer() { &lmFreeBasic, &lmFSharp, &lmGAP, + &lmGDScript, &lmGui4Cli, &lmHaskell, &lmHollywood, diff --git a/src/Lexilla/Lexilla.xcodeproj/project.pbxproj b/src/Lexilla/Lexilla.xcodeproj/project.pbxproj index 7711f75d..7976e393 100644 --- a/src/Lexilla/Lexilla.xcodeproj/project.pbxproj +++ b/src/Lexilla/Lexilla.xcodeproj/project.pbxproj @@ -155,6 +155,7 @@ 70BF497C8D265026B77C97DA /* LexJulia.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 315E4E969868C52C125686B2 /* LexJulia.cxx */; }; B32D4A2A9CEC222A5140E99F /* LexFSharp.cxx in Sources */ = {isa = PBXBuildFile; fileRef = F8E54626B22BD9493090F40B /* LexFSharp.cxx */; }; 510D44AFB91EE873E86ABDD4 /* LexAsciidoc.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 3AF14420BFC43876F16C5995 /* LexAsciidoc.cxx */; }; + 00D544CC992062D2E3CD4BF6 /* LexGDScript.cxx in Sources */ = {isa = PBXBuildFile; fileRef = A383409E9A994F461550FEC1 /* LexGDScript.cxx */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -308,6 +309,7 @@ 315E4E969868C52C125686B2 /* LexJulia.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexJulia.cxx; path = ../../lexers/LexJulia.cxx; sourceTree = SOURCE_ROOT; }; F8E54626B22BD9493090F40B /* LexFSharp.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexFSharp.cxx; path = ../../lexers/LexFSharp.cxx; sourceTree = SOURCE_ROOT; }; 3AF14420BFC43876F16C5995 /* LexAsciidoc.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexAsciidoc.cxx; path = ../../lexers/LexAsciidoc.cxx; sourceTree = SOURCE_ROOT; }; + A383409E9A994F461550FEC1 /* LexGDScript.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexGDScript.cxx; path = ../../lexers/LexGDScript.cxx; sourceTree = SOURCE_ROOT; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -396,6 +398,7 @@ 28BA72FB24E34D9400272C2D /* LexFortran.cxx */, F8E54626B22BD9493090F40B /* LexFSharp.cxx */, 28BA72F224E34D9300272C2D /* LexGAP.cxx */, + A383409E9A994F461550FEC1 /* LexGDScript.cxx */, 28BA72CF24E34D9100272C2D /* LexGui4Cli.cxx */, 28BA731B24E34D9500272C2D /* LexHaskell.cxx */, 28BA731924E34D9500272C2D /* LexHex.cxx */, @@ -724,6 +727,7 @@ B32D4A2A9CEC222A5140E99F /* LexFSharp.cxx in Sources */, 70BF497C8D265026B77C97DA /* LexJulia.cxx in Sources */, 510D44AFB91EE873E86ABDD4 /* LexAsciidoc.cxx in Sources */, + 00D544CC992062D2E3CD4BF6 /* LexGDScript.cxx in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/src/deps.mak b/src/deps.mak index 2f05c505..10c3976d 100644 --- a/src/deps.mak +++ b/src/deps.mak @@ -647,6 +647,23 @@ LexGAP.o: \ ../lexlib/StyleContext.h \ ../lexlib/CharacterSet.h \ ../lexlib/LexerModule.h +LexGDScript.o: \ + ../lexers/LexGDScript.cxx \ + ../../scintilla/include/ILexer.h \ + ../../scintilla/include/Sci_Position.h \ + ../../scintilla/include/Scintilla.h \ + ../include/SciLexer.h \ + ../lexlib/StringCopy.h \ + ../lexlib/WordList.h \ + ../lexlib/LexAccessor.h \ + ../lexlib/Accessor.h \ + ../lexlib/StyleContext.h \ + ../lexlib/CharacterSet.h \ + ../lexlib/CharacterCategory.h \ + ../lexlib/LexerModule.h \ + ../lexlib/OptionSet.h \ + ../lexlib/SubStyles.h \ + ../lexlib/DefaultLexer.h LexGui4Cli.o: \ ../lexers/LexGui4Cli.cxx \ ../../scintilla/include/ILexer.h \ diff --git a/src/lexilla.mak b/src/lexilla.mak index 5f87e9c1..ec447d3b 100644 --- a/src/lexilla.mak +++ b/src/lexilla.mak @@ -118,6 +118,7 @@ LEX_OBJS=\ $(DIR_O)\LexFortran.obj \ $(DIR_O)\LexFSharp.obj \ $(DIR_O)\LexGAP.obj \ + $(DIR_O)\LexGDScript.obj \ $(DIR_O)\LexGui4Cli.obj \ $(DIR_O)\LexHaskell.obj \ $(DIR_O)\LexHex.obj \ diff --git a/src/nmdeps.mak b/src/nmdeps.mak index eb66b102..8fc5b752 100644 --- a/src/nmdeps.mak +++ b/src/nmdeps.mak @@ -647,6 +647,23 @@ $(DIR_O)/LexGAP.obj: \ ../lexlib/StyleContext.h \ ../lexlib/CharacterSet.h \ ../lexlib/LexerModule.h +$(DIR_O)/LexGDScript.obj: \ + ../lexers/LexGDScript.cxx \ + ../../scintilla/include/ILexer.h \ + ../../scintilla/include/Sci_Position.h \ + ../../scintilla/include/Scintilla.h \ + ../include/SciLexer.h \ + ../lexlib/StringCopy.h \ + ../lexlib/WordList.h \ + ../lexlib/LexAccessor.h \ + ../lexlib/Accessor.h \ + ../lexlib/StyleContext.h \ + ../lexlib/CharacterSet.h \ + ../lexlib/CharacterCategory.h \ + ../lexlib/LexerModule.h \ + ../lexlib/OptionSet.h \ + ../lexlib/SubStyles.h \ + ../lexlib/DefaultLexer.h $(DIR_O)/LexGui4Cli.obj: \ ../lexers/LexGui4Cli.cxx \ ../../scintilla/include/ILexer.h \ diff --git a/test/examples/gdscript/AllStyles.gd b/test/examples/gdscript/AllStyles.gd new file mode 100644 index 00000000..0a484a92 --- /dev/null +++ b/test/examples/gdscript/AllStyles.gd @@ -0,0 +1,51 @@ +# Enumerate all styles: 0 to 15 +# comment=1 + +# whitespace=0 + # w + +# number=2 +37 + +# double-quoted-string=3 +"str" + +# single-quoted-string=4 +'str' + +# keyword=5 +pass + +# triple-quoted-string=6 +'''str''' + +# triple-double-quoted-string=7 +"""str""" + +# class-name=8 +class ClassName: + pass + +# function-name=9 +func function_name(): + pass + +# operator=10 +1 + 3 + +# identifier=11 +var identifier = 2 + +# comment-block=12 +## block + +# unclosed-string=13 +" unclosed + +# highlighted-identifier=14 +var hilight = 2 + +# annotation=15 +@onready +var a = 3 +@onready var b = 3 diff --git a/test/examples/gdscript/AllStyles.gd.folded b/test/examples/gdscript/AllStyles.gd.folded new file mode 100644 index 00000000..0b818f3e --- /dev/null +++ b/test/examples/gdscript/AllStyles.gd.folded @@ -0,0 +1,52 @@ + 0 400 0 # Enumerate all styles: 0 to 15 + 0 400 0 # comment=1 + 1 400 0 + 0 400 0 # whitespace=0 + 0 400 0 # w + 1 400 0 + 0 400 0 # number=2 + 0 400 0 37 + 1 400 0 + 0 400 0 # double-quoted-string=3 + 0 400 0 "str" + 1 400 0 + 0 400 0 # single-quoted-string=4 + 0 400 0 'str' + 1 400 0 + 0 400 0 # keyword=5 + 0 400 0 pass + 1 400 0 + 0 400 0 # triple-quoted-string=6 + 0 400 0 '''str''' + 1 400 0 + 0 400 0 # triple-double-quoted-string=7 + 0 400 0 """str""" + 1 400 0 + 0 400 0 # class-name=8 + 2 400 0 + class ClassName: + 0 408 0 | pass + 1 400 0 + 0 400 0 # function-name=9 + 2 400 0 + func function_name(): + 0 408 0 | pass + 1 400 0 + 0 400 0 # operator=10 + 0 400 0 1 + 3 + 1 400 0 + 0 400 0 # identifier=11 + 0 400 0 var identifier = 2 + 1 400 0 + 0 400 0 # comment-block=12 + 0 400 0 ## block + 1 400 0 + 0 400 0 # unclosed-string=13 + 0 400 0 " unclosed + 1 400 0 + 0 400 0 # highlighted-identifier=14 + 0 400 0 var hilight = 2 + 1 400 0 + 0 400 0 # annotation=15 + 0 400 0 @onready + 0 400 0 var a = 3 + 0 400 0 @onready var b = 3 + 1 400 0 \ No newline at end of file diff --git a/test/examples/gdscript/AllStyles.gd.styled b/test/examples/gdscript/AllStyles.gd.styled new file mode 100644 index 00000000..42f3a48c --- /dev/null +++ b/test/examples/gdscript/AllStyles.gd.styled @@ -0,0 +1,51 @@ +{1}# Enumerate all styles: 0 to 15{0} +{1}# comment=1{0} + +{1}# whitespace=0{0} + {1}# w{0} + +{1}# number=2{0} +{2}37{0} + +{1}# double-quoted-string=3{0} +{3}"str"{0} + +{1}# single-quoted-string=4{0} +{4}'str'{0} + +{1}# keyword=5{0} +{5}pass{0} + +{1}# triple-quoted-string=6{0} +{6}'''str'''{0} + +{1}# triple-double-quoted-string=7{0} +{7}"""str"""{0} + +{1}# class-name=8{0} +{5}class{0} {8}ClassName{10}:{0} + {5}pass{0} + +{1}# function-name=9{0} +{5}func{0} {9}function_name{10}():{0} + {5}pass{0} + +{1}# operator=10{0} +{2}1{0} {10}+{0} {2}3{0} + +{1}# identifier=11{0} +{5}var{0} {11}identifier{0} {10}={0} {2}2{0} + +{1}# comment-block=12{0} +{12}## block{0} + +{1}# unclosed-string=13{0} +{13}" unclosed +{0} +{1}# highlighted-identifier=14{0} +{5}var{0} {14}hilight{0} {10}={0} {2}2{0} + +{1}# annotation=15{0} +{15}@onready{0} +{5}var{0} {11}a{0} {10}={0} {2}3{0} +{15}@onready{0} {5}var{0} {11}b{0} {10}={0} {2}3{0} diff --git a/test/examples/gdscript/SciTE.properties b/test/examples/gdscript/SciTE.properties new file mode 100644 index 00000000..21a1ec49 --- /dev/null +++ b/test/examples/gdscript/SciTE.properties @@ -0,0 +1,5 @@ +lexer.*.gd=gdscript +keywords.*.gd=class func else for if extends in pass print return while var +keywords2.*.gd=hilight +fold=1 +fold.compact=1