diff --git a/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF b/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF index ce370dd966..e8a22ef134 100644 --- a/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF +++ b/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF @@ -45,6 +45,9 @@ Export-Package: org.eclipse.xtext.xtext.generator, org.eclipse.xtext.xbase, org.eclipse.xtext.tests, org.eclipse.xtend.core", + org.eclipse.xtext.xtext.generator.parser.antlr.ex;x-friends:="org.eclipse.emf.mwe2.language, + org.eclipse.xtext.ui.codetemplates, + org.eclipse.xtext.ui.tests", org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing;x-internal:=true, org.eclipse.xtext.xtext.generator.parser.antlr.splitting;x-friends:="org.eclipse.xtext.generator, org.eclipse.xtext.eclipse.tests, diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend index 5899ccfcfe..ea006effcb 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend @@ -22,6 +22,7 @@ import org.eclipse.xtext.ParserRule import org.eclipse.xtext.UnorderedGroup import org.eclipse.xtext.generator.LineSeparatorHarmonizer import org.eclipse.xtext.util.Strings +import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment import org.eclipse.xtext.xtext.generator.CodeConfig import org.eclipse.xtext.xtext.generator.Issues import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess @@ -31,11 +32,10 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrCodeQuality import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrParserSplitter import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardForUnorderedGroupsRemover +import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.PartialClassExtractor import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.SyntacticPredicateFixup import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.UnorderedGroupsSplitter -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover -import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFragment { @Inject @Accessors(PROTECTED_GETTER) AntlrToolFacade antlrTool @@ -169,11 +169,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra } } - def protected improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) { - var lexerContent = fsa.readTextFile(lexer.javaPath).toString - lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options) - fsa.generateFile(lexer.javaPath, lexerContent) - + def protected void improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) { + improveLexerCodeQuality(fsa, lexer) + improveParserCodeQuality(fsa, parser) + } + + def protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { var parserContent = fsa.readTextFile(parser.javaPath).toString parserContent = codeQualityHelper.stripUnnecessaryComments(parserContent, options) parserContent = codeQualityHelper.removeDuplicateBitsets(parserContent, options) @@ -181,6 +182,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra fsa.generateFile(parser.javaPath, parserContent) } + def protected void improveLexerCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer) { + var lexerContent = fsa.readTextFile(lexer.javaPath).toString + lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options) + fsa.generateFile(lexer.javaPath, lexerContent) + } + def protected void cleanupLexerTokensFile(AntlrGrammar lexerGrammar, KeywordHelper helper, IXtextGeneratorFileSystemAccess fsa) { if (options.backtrackLexer) { val provider = createLexerTokensProvider(lexerGrammar, helper, fsa) diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java new file mode 100644 index 0000000000..3567501c71 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java @@ -0,0 +1,178 @@ +/******************************************************************************* + * Copyright (c) 2019 itemis AG (http://www.itemis.eu) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.ex; + +import org.eclipse.xtend2.lib.StringConcatenationClient; +import org.eclipse.xtext.parser.antlr.Lexer; +import org.eclipse.xtext.xtext.generator.Issues; +import org.eclipse.xtext.xtext.generator.model.GuiceModuleAccess; +import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess; +import org.eclipse.xtext.xtext.generator.model.TypeReference; +import org.eclipse.xtext.xtext.generator.parser.antlr.AbstractAntlrGeneratorFragment2; + +import com.google.common.annotations.Beta; + +/** + * Use this fragment if you want to use an external lexer , e.g: in case the generated lexer has to be customized. + * + * @since 2.20 + * @author dietrich - Initial contribution and API + */ +@Beta +public class ExternalAntlrLexerFragment extends AbstractAntlrGeneratorFragment2 { + + private String lexerGrammar; + + private boolean highlighting; + + private boolean runtime; + + private boolean contentAssist; + + @Override + protected void doGenerate() { + if (runtime) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(" + Lexer.class.getName() + ".class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.parser.antlr.LexerBindings.RUNTIME" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("RuntimeLexer", binding).contributeTo(getLanguage().getRuntimeGenModule()); + } + if (highlighting) { + if (this.getProjectConfig().getEclipsePlugin().getRoot() != null) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(" + Lexer.class.getName() + ".class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.ide.LexerIdeBindings.HIGHLIGHTING" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("HighlightingLexer", binding).contributeTo(getLanguage().getEclipsePluginGenModule()); + } + } + if (contentAssist) { + if (this.getProjectConfig().getGenericIde().getRoot() != null) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(org.eclipse.xtext.ide.editor.contentassist.antlr.internal.Lexer.class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.ide.LexerIdeBindings.CONTENT_ASSIST" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("ContentAssistLexer", binding).contributeTo(getLanguage().getIdeGenModule()); + + } + if (this.getProjectConfig().getEclipsePlugin().getRoot() != null) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(org.eclipse.xtext.ide.editor.contentassist.antlr.internal.Lexer.class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.ide.LexerIdeBindings.CONTENT_ASSIST" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("ContentAssistLexer", binding).contributeTo(getLanguage().getEclipsePluginGenModule()); + } + } + + + + + IXtextGeneratorFileSystemAccess srcGenFsa = this.getProjectConfig().getRuntime().getSrcGen(); + IXtextGeneratorFileSystemAccess srcFsa = this.getProjectConfig().getRuntime().getSrc(); + if (contentAssist || highlighting) { + if (this.getProjectConfig().getGenericIde().getRoot() != null) { + srcGenFsa = this.getProjectConfig().getGenericIde().getSrcGen(); + srcFsa = this.getProjectConfig().getGenericIde().getSrc(); + } else { + srcGenFsa = this.getProjectConfig().getEclipsePlugin().getSrcGen(); + srcFsa = this.getProjectConfig().getEclipsePlugin().getSrc(); + } + } + String srcGenPath = srcGenFsa.getPath(); + String srcPath = srcFsa.getPath(); + String grammarFile = srcPath + "/" + getLexerGrammar().replace('.', '/') + ".g"; + String generateTo = ""; + if (getLexerGrammar().lastIndexOf('.') != -1) { + generateTo = getLexerGrammar().substring(0, getLexerGrammar().lastIndexOf('.')); + } + generateTo = srcGenPath + "/" + generateTo.replace('.', '/'); + addAntlrParam("-fo"); + addAntlrParam(generateTo); + final String encoding = getCodeConfig().getEncoding(); + getAntlrTool().runWithEncodingAndParams(grammarFile, encoding, getAntlrParams()); + + TypeReference lexerType = new TypeReference(getLexerGrammar()); + splitParserAndLexerIfEnabled(srcGenFsa, null /* parser */, lexerType); + normalizeTokens(srcGenFsa, getLexerGrammar().replace('.', '/') + ".tokens"); + suppressWarnings(srcGenFsa, lexerType); + normalizeLineDelimiters(srcGenFsa, lexerType); + } + + @Override + protected void splitParserClassFile(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { + // no-op + } + + @Override + protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { + // no-op + } + + @Override + public void checkConfiguration(Issues issues) { + super.checkConfiguration(issues); + if (contentAssist && highlighting || runtime && highlighting || contentAssist && runtime) { + issues.addError("Only one of those flags is allowed: contentAssist, runtime, highlighting flag"); + } + } + + + public void setLexerGrammar(String lexerGrammar) { + this.lexerGrammar = lexerGrammar; + } + + public String getLexerGrammar() { + return lexerGrammar; + } + + public void setHighlighting(boolean highlighting) { + this.highlighting = highlighting; + } + + public boolean isHighlighting() { + return highlighting; + } + + public void setRuntime(boolean runtime) { + this.runtime = runtime; + } + + public boolean isRuntime() { + return runtime; + } + + public void setContentAssist(boolean contentAssist) { + this.contentAssist = contentAssist; + } + + public boolean isContentAssist() { + return contentAssist; + } + +} diff --git a/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java b/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java index d88f4d79bf..d04b99048a 100644 --- a/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java +++ b/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java @@ -219,9 +219,11 @@ protected void splitParserAndLexerIfEnabled(final IXtextGeneratorFileSystemAcces } protected void improveCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer, final TypeReference parser) { - String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString(); - lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options); - fsa.generateFile(lexer.getJavaPath(), lexerContent); + this.improveLexerCodeQuality(fsa, lexer); + this.improveParserCodeQuality(fsa, parser); + } + + protected void improveParserCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference parser) { String parserContent = fsa.readTextFile(parser.getJavaPath()).toString(); parserContent = this.codeQualityHelper.stripUnnecessaryComments(parserContent, this.options); parserContent = this.codeQualityHelper.removeDuplicateBitsets(parserContent, this.options); @@ -229,6 +231,12 @@ protected void improveCodeQuality(final IXtextGeneratorFileSystemAccess fsa, fin fsa.generateFile(parser.getJavaPath(), parserContent); } + protected void improveLexerCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer) { + String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString(); + lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options); + fsa.generateFile(lexer.getJavaPath(), lexerContent); + } + protected void cleanupLexerTokensFile(final AntlrGrammar lexerGrammar, final KeywordHelper helper, final IXtextGeneratorFileSystemAccess fsa) { try { boolean _isBacktrackLexer = this.options.isBacktrackLexer();