Split grammar and main export (#1171)

eclipse-langium · Dec 1, 2023 · 729eeed · 729eeed
1 parent 750ff44
commit 729eeed
Show file tree

Hide file tree

Showing 109 changed files with 637 additions and 537 deletions.
diff --git a/packages/langium-cli/src/generate.ts b/packages/langium-cli/src/generate.ts
@@ -8,8 +8,8 @@ import type { AstNode, Grammar, LangiumDocument, Mutable } from 'langium';
 import type { LangiumConfig, LangiumLanguageConfig} from './package.js';
 import { URI } from 'langium';
 import { loadConfig } from './package.js';
-import { copyAstNode, createLangiumGrammarServices, getDocument, GrammarAST, linkContentToContainer } from 'langium';
-import { resolveImport, resolveTransitiveImports } from 'langium/internal';
+import { copyAstNode, getDocument, GrammarAST, linkContentToContainer } from 'langium';
+import { createLangiumGrammarServices, resolveImport, resolveTransitiveImports } from 'langium/grammar';
 import { NodeFileSystem } from 'langium/node';
 import { generateAst } from './generator/ast-generator.js';
 import { serializeGrammar } from './generator/grammar-serializer.js';
@@ -332,7 +332,7 @@ export async function runGenerator(config: LangiumConfig, options: GenerateOptio
     }
 
     const genAst = generateAst(grammarServices, embeddedGrammars, config);
-    await writeWithFail(path.resolve(output, 'ast.ts'), genAst, options);
+    await writeWithFail(path.resolve(updateLangiumInternalAstPath(output, config), 'ast.ts'), genAst, options);
 
     const serializedGrammar = serializeGrammar(grammarServices, embeddedGrammars, config);
     await writeWithFail(path.resolve(output, 'grammar.ts'), serializedGrammar, options);
@@ -393,6 +393,16 @@ export async function runGenerator(config: LangiumConfig, options: GenerateOptio
     return buildResult(true);
 }
 
+function updateLangiumInternalAstPath(output: string, config: LangiumConfig): string {
+    if (config.langiumInternal) {
+        // The Langium internal ast is generated to the languages package.
+        // This is done to prevent internal access to the `langium/grammar` export.
+        return path.join(output, '..', '..', 'languages', 'generated');
+    } else {
+        return output;
+    }
+}
+
 export async function generateTypes(options: ExtractTypesOptions): Promise<void> {
     const grammarPath = path.isAbsolute(options.grammar) ? options.grammar : path.resolve('.', options.grammar);
     if (!fs.existsSync(grammarPath) || !fs.lstatSync(grammarPath).isFile()) {

diff --git a/packages/langium-cli/src/generator/ast-generator.ts b/packages/langium-cli/src/generator/ast-generator.ts
@@ -5,10 +5,10 @@
  ******************************************************************************/
 import type { Grammar, LangiumServices } from 'langium';
 import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate';
-import type { AstTypes, Property } from 'langium/types';
+import type { AstTypes, Property } from 'langium/grammar';
 import type { LangiumConfig } from '../package.js';
 import { streamAllContents, MultiMap, GrammarAST } from 'langium';
-import { collectAst, collectTypeHierarchy, findReferenceTypes, hasArrayType, isAstType, hasBooleanType, mergeTypesAndInterfaces } from 'langium/types';
+import { collectAst, collectTypeHierarchy, findReferenceTypes, hasArrayType, isAstType, hasBooleanType, mergeTypesAndInterfaces } from 'langium/grammar';
 import { collectTerminalRegexps, generatedHeader } from './util.js';
 
 export function generateAst(services: LangiumServices, grammars: Grammar[], config: LangiumConfig): string {

diff --git a/packages/langium-cli/src/generator/grammar-serializer.ts b/packages/langium-cli/src/generator/grammar-serializer.ts
@@ -16,8 +16,8 @@ export function serializeGrammar(services: LangiumServices, grammars: Grammar[],
     `.appendNewLine(
     ).appendTemplateIf(!!config.langiumInternal)`
 
-        import type { Grammar } from './ast${config.importExtension}';
-        import { loadGrammarFromJson } from '../../utils/grammar-util${config.importExtension}';
+        import type { Grammar } from '../../languages/generated/ast${config.importExtension}';
+        import { loadGrammarFromJson } from '../../utils/grammar-loader${config.importExtension}';
     `.appendTemplateIf(!config.langiumInternal)`
 
         import type { Grammar } from 'langium';

diff --git a/packages/langium-cli/src/generator/highlighting/monarch-generator.ts b/packages/langium-cli/src/generator/highlighting/monarch-generator.ts
@@ -4,9 +4,8 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-import { type Grammar, getTerminalParts, isCommentTerminal, escapeRegExp, GrammarAST, isWhitespaceRegExp } from 'langium';
+import { type Grammar, getTerminalParts, isCommentTerminal, escapeRegExp, GrammarAST, isWhitespaceRegExp, terminalRegex } from 'langium';
 import { type Generated, expandToNode, joinToNode, toString } from 'langium/generate';
-import { terminalRegex } from 'langium/internal';
 import type { LangiumLanguageConfig } from '../../package.js';
 import { collectKeywords } from '../util.js';
 

diff --git a/packages/langium-cli/src/generator/highlighting/prism-generator.ts b/packages/langium-cli/src/generator/highlighting/prism-generator.ts
@@ -3,9 +3,8 @@
  * This program and the accompanying materials are made available under the
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
-import { GrammarAST, escapeRegExp, isCommentTerminal, type Grammar } from 'langium';
+import { terminalRegex, GrammarAST, escapeRegExp, isCommentTerminal, type Grammar } from 'langium';
 import { expandToNode, joinToNode, toString, type Generated } from 'langium/generate';
-import { terminalRegex } from 'langium/internal';
 import _ from 'lodash';
 import type { LangiumLanguageConfig } from '../../package.js';
 import { collectKeywords } from '../util.js';

diff --git a/packages/langium-cli/src/generator/highlighting/textmate-generator.ts b/packages/langium-cli/src/generator/highlighting/textmate-generator.ts
@@ -4,8 +4,7 @@
  * terms of the MIT License, which is available in the project root.
 ******************************************************************************/
 import type { Grammar } from 'langium';
-import { escapeRegExp, getCaseInsensitivePattern, getTerminalParts, GrammarAST, isCommentTerminal, stream } from 'langium';
-import { terminalRegex } from 'langium/internal';
+import { escapeRegExp, getCaseInsensitivePattern, getTerminalParts, GrammarAST, isCommentTerminal, stream, terminalRegex } from 'langium';
 import type { LangiumLanguageConfig } from '../../package.js';
 import { collectKeywords } from '../util.js';
 

diff --git a/packages/langium-cli/src/generator/module-generator.ts b/packages/langium-cli/src/generator/module-generator.ts
@@ -22,7 +22,8 @@ export function generateModule(grammars: Grammar[], config: LangiumConfig, gramm
     ).appendIf(!!config.langiumInternal,
         expandToNode`
 
-            import type { LanguageMetaData } from '../language-meta-data${config.importExtension}';
+            import type { LanguageMetaData } from '../../languages/language-meta-data${config.importExtension}';
+            import { ${config.projectName}AstReflection } from '../../languages/generated/ast${config.importExtension}';
             import type { Module } from '../../dependency-injection${config.importExtension}';
             import type { LangiumGeneratedServices, LangiumGeneratedSharedServices, LangiumSharedServices, LangiumServices } from '../../services${config.importExtension}';
         `.appendTemplateIf(hasIParserConfigImport)`
@@ -32,9 +33,9 @@ export function generateModule(grammars: Grammar[], config: LangiumConfig, gramm
     ).appendTemplateIf(!config.langiumInternal)`
 
         import type { LangiumGeneratedServices, LangiumGeneratedSharedServices, LangiumSharedServices, LangiumServices, LanguageMetaData, Module${hasIParserConfigImport ? ', IParserConfig' : ''} } from 'langium';
+        import { ${config.projectName}AstReflection } from './ast${config.importExtension}';
     `.appendTemplate`
 
-        import { ${config.projectName}AstReflection } from './ast${config.importExtension}';
         import { ${joinToNode(grammarsWithName, grammar => grammar.name + 'Grammar', { separator: ', '}) } } from './grammar${config.importExtension}';
         ${joinToNode(
             grammarsWithName,

diff --git a/packages/langium-cli/src/generator/types-generator.ts b/packages/langium-cli/src/generator/types-generator.ts
@@ -5,8 +5,7 @@
  ******************************************************************************/
 import type { Grammar, LangiumServices } from 'langium';
 import { joinToNode, toString } from 'langium/generate';
-import { collectAst } from 'langium/types';
-import { LangiumGrammarGrammar } from 'langium/internal';
+import { collectAst, LangiumGrammarGrammar } from 'langium/grammar';
 import { collectKeywords } from './util.js';
 
 export function generateTypesFile(services: LangiumServices, grammars: Grammar[]): string {

diff --git a/packages/langium-cli/src/generator/util.ts b/packages/langium-cli/src/generator/util.ts
@@ -3,14 +3,13 @@
  * This program and the accompanying materials are made available under the
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
-import { type Grammar, getAllReachableRules, GrammarAST, stream, streamAllContents } from 'langium';
+import { type Grammar, getAllReachableRules, GrammarAST, stream, streamAllContents, terminalRegex } from 'langium';
 import { type Generated, expandToNode } from 'langium/generate';
 import fs from 'fs-extra';
 import * as path from 'node:path';
 import * as url from 'node:url';
 import * as readline from 'node:readline';
 import chalk from 'chalk';
-import { terminalRegex } from 'langium/internal';
 
 // This is a replacement for `__dirname`
 function getDirname(): string {

diff --git a/packages/langium-cli/src/parser-validation.ts b/packages/langium-cli/src/parser-validation.ts
@@ -3,9 +3,11 @@
  * This program and the accompanying materials are made available under the
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
-import type { Grammar, IParserConfig, LangiumDocuments, LangiumGrammarServices, LangiumParser, LanguageMetaData, } from 'langium';
+import type { Grammar, IParserConfig, LangiumDocuments, LangiumParser, LanguageMetaData } from 'langium';
 import type { LangiumConfig, LangiumLanguageConfig } from './package.js';
-import { createServicesForGrammar, getDocument, GrammarAST, prepareLangiumParser } from 'langium';
+import { getDocument, GrammarAST, prepareLangiumParser } from 'langium';
+import type { LangiumGrammarServices} from 'langium/grammar';
+import { createServicesForGrammar } from 'langium/grammar';
 import { getFilePath } from './package.js';
 
 export async function validateParser(grammar: Grammar, config: LangiumConfig, grammarConfigMap: Map<Grammar, LangiumLanguageConfig>,

diff --git a/packages/langium-cli/test/generator/ast-generator.test.ts b/packages/langium-cli/test/generator/ast-generator.test.ts
@@ -4,9 +4,10 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-import { EmptyFileSystem, createLangiumGrammarServices, type Grammar } from 'langium';
+import { EmptyFileSystem, type Grammar } from 'langium';
 import { expandToString, normalizeEOL } from 'langium/generate';
 import { parseHelper } from 'langium/test';
+import { createLangiumGrammarServices } from 'langium/grammar';
 import { describe, expect, test } from 'vitest';
 import { generateAst } from '../../src/generator/ast-generator.js';
 import type { LangiumConfig } from '../../src/package.js';

diff --git a/packages/langium-cli/test/generator/types-generator.test.ts b/packages/langium-cli/test/generator/types-generator.test.ts
@@ -5,8 +5,9 @@
  ******************************************************************************/
 
 import type { Grammar } from 'langium';
-import { EmptyFileSystem, createLangiumGrammarServices } from 'langium';
+import { EmptyFileSystem } from 'langium';
 import { expandToStringWithNL } from 'langium/generate';
+import { createLangiumGrammarServices } from 'langium/grammar';
 import { parseHelper } from 'langium/test';
 import { describe, expect, test } from 'vitest';
 import { generateTypesFile } from '../../src/generator/types-generator.js';

diff --git a/packages/langium-vscode/src/language-server/main.ts b/packages/langium-vscode/src/language-server/main.ts
@@ -5,7 +5,8 @@
  ******************************************************************************/
 
 import type { LangiumSharedServices, Module, PartialLangiumSharedServices } from 'langium';
-import { createLangiumGrammarServices, startLanguageServer } from 'langium';
+import { startLanguageServer } from 'langium';
+import { createLangiumGrammarServices } from 'langium/grammar';
 import { NodeFileSystem } from 'langium/node';
 import { createConnection, ProposedFeatures } from 'vscode-languageserver/node.js';
 import { LangiumGrammarWorkspaceManager } from './grammar-workspace-manager.js';

diff --git a/packages/langium-vscode/src/language-server/railroad-handler.ts b/packages/langium-vscode/src/language-server/railroad-handler.ts
@@ -8,7 +8,7 @@ import type { Grammar, LangiumServices } from 'langium';
 import { DocumentState, GrammarAST, URI } from 'langium';
 import { createGrammarDiagramHtml } from 'langium-railroad';
 import { expandToString } from 'langium/generate';
-import { resolveTransitiveImports } from 'langium/internal';
+import { resolveTransitiveImports } from 'langium/grammar';
 import type { Connection } from 'vscode-languageserver';
 import { DiagnosticSeverity } from 'vscode-languageserver';
 import { DOCUMENTS_VALIDATED_NOTIFICATION, RAILROAD_DIAGRAM_REQUEST } from './messages.js';

diff --git a/packages/langium/package.json b/packages/langium/package.json
@@ -42,13 +42,9 @@
       "types": "./lib/node/index.d.ts",
       "import": "./lib/node/index.js"
     },
-    "./types": {
-      "types": "./lib/grammar/type-system/index.d.ts",
-      "import": "./lib/grammar/type-system/index.js"
-    },
-    "./internal": {
-      "types": "./lib/grammar/internal-grammar-util.d.ts",
-      "import": "./lib/grammar/internal-grammar-util.js"
+    "./grammar": {
+      "types": "./lib/grammar/index.d.ts",
+      "import": "./lib/grammar/index.js"
     }
   },
   "scripts": {

diff --git a/packages/langium/src/default-module.ts b/packages/langium/src/default-module.ts
@@ -10,7 +10,7 @@ import type { LangiumDefaultServices, LangiumDefaultSharedServices, LangiumServi
 import type { FileSystemProvider } from './workspace/file-system-provider.js';
 import { TextDocuments } from 'vscode-languageserver';
 import { TextDocument } from 'vscode-languageserver-textdocument';
-import { createGrammarConfig } from './grammar/grammar-config.js';
+import { createGrammarConfig } from './languages/grammar-config.js';
 import { createCompletionParser } from './parser/completion-parser-builder.js';
 import { DefaultCompletionProvider } from './lsp/completion/completion-provider.js';
 import { DefaultDocumentHighlightProvider } from './lsp/document-highlight-provider.js';

diff --git a/packages/langium/src/documentation/comment-provider.ts b/packages/langium/src/documentation/comment-provider.ts
@@ -4,7 +4,7 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-import type { GrammarConfig } from '../grammar/grammar-config.js';
+import type { GrammarConfig } from '../languages/grammar-config.js';
 import { isAstNodeWithComment } from '../serializer/json-serializer.js';
 import type { LangiumServices } from '../services.js';
 import type { AstNode } from '../syntax-tree.js';

diff --git a/packages/langium/src/grammar/ast-reflection-interpreter.ts b/packages/langium/src/grammar/ast-reflection-interpreter.ts
@@ -6,11 +6,11 @@
 
 import type { AstReflection, ReferenceInfo, TypeMandatoryProperty, TypeMetaData } from '../syntax-tree.js';
 import type { LangiumDocuments } from '../workspace/documents.js';
-import type { Grammar } from './generated/ast.js';
+import type { Grammar } from '../languages/generated/ast.js';
 import type { AstTypes, Property } from './type-system/type-collector/types.js';
 import { AbstractAstReflection } from '../syntax-tree.js';
 import { MultiMap } from '../utils/collections.js';
-import { isGrammar } from './generated/ast.js';
+import { isGrammar } from '../languages/generated/ast.js';
 import { collectAst } from './type-system/ast-collector.js';
 import { collectTypeHierarchy, findReferenceTypes, hasArrayType, isAstType, hasBooleanType, mergeTypesAndInterfaces } from './type-system/types-util.js';
 

diff --git a/packages/langium/src/grammar/generated/grammar.ts b/packages/langium/src/grammar/generated/grammar.ts
@@ -3,8 +3,8 @@
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 
-import type { Grammar } from './ast.js';
-import { loadGrammarFromJson } from '../../utils/grammar-util.js';
+import type { Grammar } from '../../languages/generated/ast.js';
+import { loadGrammarFromJson } from '../../utils/grammar-loader.js';
 
 let loadedLangiumGrammarGrammar: Grammar | undefined;
 export const LangiumGrammarGrammar = (): Grammar => loadedLangiumGrammarGrammar ?? (loadedLangiumGrammarGrammar = loadGrammarFromJson(`{

diff --git a/packages/langium/src/grammar/generated/module.ts b/packages/langium/src/grammar/generated/module.ts
@@ -3,11 +3,11 @@
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 
-import type { LanguageMetaData } from '../language-meta-data.js';
+import type { LanguageMetaData } from '../../languages/language-meta-data.js';
+import { LangiumGrammarAstReflection } from '../../languages/generated/ast.js';
 import type { Module } from '../../dependency-injection.js';
 import type { LangiumGeneratedServices, LangiumGeneratedSharedServices, LangiumSharedServices, LangiumServices } from '../../services.js';
 import type { IParserConfig } from '../../parser/parser-config.js';
-import { LangiumGrammarAstReflection } from './ast.js';
 import { LangiumGrammarGrammar } from './grammar.js';
 
 export const LangiumGrammarLanguageMetaData = {

diff --git a/packages/langium/src/grammar/index.ts b/packages/langium/src/grammar/index.ts
@@ -4,7 +4,26 @@
  * terms of the MIT License, which is available in the project root.
  ******************************************************************************/
 
-export * from './ast-reflection-interpreter.js';
-export * from './grammar-config.js';
+// This file contains Langium grammar language internals.
+// It is not supposed to be exported with the general `langium` export.
+// Instead, it is available from `langium/grammar`.
+
+export * from './generated/grammar.js';
+export * from './generated/module.js';
+export * from './lsp/grammar-call-hierarchy.js';
+export * from './lsp/grammar-code-actions.js';
+export * from './lsp/grammar-completion-provider.js';
+export * from './lsp/grammar-definition.js';
+export * from './lsp/grammar-folding-ranges.js';
+export * from './lsp/grammar-formatter.js';
+export * from './lsp/grammar-semantic-tokens.js';
+export * from './references/grammar-naming.js';
+export * from './references/grammar-references.js';
+export * from './references/grammar-scope.js';
+export * from './validation/types-validator.js';
+export * from './validation/validation-resources-collector.js';
+export * from './validation/validator.js';
+export * from './type-system/index.js';
 export * from './langium-grammar-module.js';
-export * from './language-meta-data.js';
+export * from './internal-grammar-util.js';
+export * from './ast-reflection-interpreter.js';