add AST strong types (#719)

Adds TypeScript AST types, and some initial tests. Additionally: - changed the DSL `Item` to be an enum of `Rc<>` variants, instead of an `Rc<>` to an enum, to make it easier to clone/store individual variants separately. - unified some inconsistent Rust/TypeScript node helpers: `*_with_kind()`, `*_with_kinds()`, `*_is_kind()`), ...
NomicFoundation · Dec 23, 2023 · 1ad6bb3 · 1ad6bb3
1 parent 995cdb0
commit 1ad6bb3
Show file tree

Hide file tree

Showing 73 changed files with 13,362 additions and 332 deletions.
diff --git a/.changeset/quick-mirrors-film.md b/.changeset/quick-mirrors-film.md
@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": patch
+---
+
+unify Rust/TypeScript node helpers: `*_with_kind()`, `*_with_kinds()`, `*_is_kind()`), ...
diff --git a/.changeset/tall-melons-happen.md b/.changeset/tall-melons-happen.md
@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": minor
+---
+
+introduce strong types for all Solidity non terminals in the TypeScript API.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/codegen/language/definition/src/compiler/analysis/definitions.rs b/crates/codegen/language/definition/src/compiler/analysis/definitions.rs
@@ -41,7 +41,7 @@ fn collect_top_level_items(analysis: &mut Analysis) {
 
 fn check_enum_items(analysis: &mut Analysis) {
     for item in analysis.language.clone().items() {
-        let SpannedItem::Enum { item } = item.as_ref() else {
+        let SpannedItem::Enum { item } = item else {
             continue;
         };
 
@@ -64,7 +64,7 @@ fn check_precedence_items(analysis: &mut Analysis) {
     let mut all_expressions = HashSet::new();
 
     for item in analysis.language.clone().items() {
-        let SpannedItem::Precedence { item } = item.as_ref() else {
+        let SpannedItem::Precedence { item } = item else {
             continue;
         };
 

diff --git a/crates/codegen/language/definition/src/compiler/analysis/mod.rs b/crates/codegen/language/definition/src/compiler/analysis/mod.rs
@@ -23,7 +23,7 @@ pub(crate) struct Analysis {
 
 pub(crate) struct ItemMetadata {
     pub name: Spanned<Identifier>,
-    pub item: Rc<SpannedItem>,
+    pub item: SpannedItem,
 
     pub defined_in: VersionSet,
     pub used_in: VersionSet,
@@ -68,7 +68,7 @@ impl Analysis {
 }
 
 impl SpannedLanguage {
-    fn items(&self) -> impl Iterator<Item = &Rc<SpannedItem>> {
+    fn items(&self) -> impl Iterator<Item = &SpannedItem> {
         return self
             .sections
             .iter()

diff --git a/crates/codegen/language/definition/src/compiler/analysis/reachability.rs b/crates/codegen/language/definition/src/compiler/analysis/reachability.rs
@@ -64,8 +64,8 @@ fn collect_trivia<'l>(parser: &'l SpannedTriviaParser, acc: &mut Vec<&'l Identif
         | SpannedTriviaParser::Optional { parser } => {
             collect_trivia(parser, acc);
         }
-        SpannedTriviaParser::Trivia { trivia } => {
-            acc.push(trivia);
+        SpannedTriviaParser::Trivia { reference } => {
+            acc.push(reference);
         }
     };
 }

diff --git a/crates/codegen/language/definition/src/compiler/analysis/references.rs b/crates/codegen/language/definition/src/compiler/analysis/references.rs
@@ -114,7 +114,7 @@ fn check_enum(analysis: &mut Analysis, item: &SpannedEnumItem, enablement: &Vers
 fn check_repeated(analysis: &mut Analysis, item: &SpannedRepeatedItem, enablement: &VersionSet) {
     let SpannedRepeatedItem {
         name,
-        repeated,
+        reference,
         enabled,
     } = item;
 
@@ -123,7 +123,7 @@ fn check_repeated(analysis: &mut Analysis, item: &SpannedRepeatedItem, enablemen
     check_reference(
         analysis,
         Some(name),
-        repeated,
+        reference,
         &enablement,
         &[
             Struct, Enum, Repeated, Separated, Precedence, Keyword, Token,
@@ -134,7 +134,7 @@ fn check_repeated(analysis: &mut Analysis, item: &SpannedRepeatedItem, enablemen
 fn check_separated(analysis: &mut Analysis, item: &SpannedSeparatedItem, enablement: &VersionSet) {
     let SpannedSeparatedItem {
         name,
-        separated,
+        reference,
         separator,
         enabled,
     } = item;
@@ -144,7 +144,7 @@ fn check_separated(analysis: &mut Analysis, item: &SpannedSeparatedItem, enablem
     check_reference(
         analysis,
         Some(name),
-        separated,
+        reference,
         &enablement,
         &[
             Struct, Enum, Repeated, Separated, Precedence, Keyword, Token,
@@ -169,7 +169,7 @@ fn check_precedence(
     let enablement = update_enablement(analysis, enablement, enabled);
 
     for precedence_expression in precedence_expressions {
-        let SpannedPrecedenceExpression { name: _, operators } = precedence_expression;
+        let SpannedPrecedenceExpression { name: _, operators } = precedence_expression.as_ref();
 
         for operator in operators {
             let SpannedPrecedenceOperator {
@@ -230,8 +230,6 @@ fn check_fields(
                     reference,
                     &enablement,
                     &[
-                        // TODO(#638): remove [Separated] and [Repeated] from here, once they are allowed to be empty.
-                        // Therefore, we ensure we always produce the parent node, even if it has no children.
                         Struct, Enum, Repeated, Separated, Precedence, Keyword, Token,
                     ],
                 );
@@ -256,8 +254,8 @@ fn check_trivia_parser(
         | SpannedTriviaParser::Optional { parser } => {
             check_trivia_parser(analysis, parser, enablement);
         }
-        SpannedTriviaParser::Trivia { trivia } => {
-            check_reference(analysis, None, trivia, enablement, &[Trivia]);
+        SpannedTriviaParser::Trivia { reference } => {
+            check_reference(analysis, None, reference, enablement, &[Trivia]);
         }
     };
 }
@@ -376,7 +374,7 @@ fn check_reference(
         );
     }
 
-    let actual_kind: SpannedItemDiscriminants = target.item.as_ref().into();
+    let actual_kind: SpannedItemDiscriminants = target.item.clone().into();
     if !expected_kinds.contains(&actual_kind) {
         analysis.errors.add(
             reference,

diff --git a/crates/codegen/language/definition/src/internals/parse_input_tokens/external_types.rs b/crates/codegen/language/definition/src/internals/parse_input_tokens/external_types.rs
@@ -24,6 +24,12 @@ impl<T: ParseInputTokens> ParseInputTokens for Box<T> {
 
         Ok(value.into())
     }
+
+    fn parse_named_value(input: ParseStream<'_>, errors: &mut ErrorsCollection) -> Result<Self> {
+        let value = T::parse_named_value(input, errors)?;
+
+        Ok(value.into())
+    }
 }
 
 impl ParseInputTokens for char {
@@ -68,6 +74,13 @@ impl<T: ParseInputTokens> ParseInputTokens for Option<T> {
             Ok(Some(T::parse_value(input, errors)?))
         }
     }
+    fn parse_named_value(input: ParseStream<'_>, errors: &mut ErrorsCollection) -> Result<Self> {
+        if input.is_empty() {
+            Ok(None)
+        } else {
+            Ok(Some(T::parse_named_value(input, errors)?))
+        }
+    }
 
     fn parse_field(
         name: &str,
@@ -87,6 +100,12 @@ impl<T: ParseInputTokens> ParseInputTokens for Rc<T> {
 
         Ok(value.into())
     }
+
+    fn parse_named_value(input: ParseStream<'_>, errors: &mut ErrorsCollection) -> Result<Self> {
+        let value = T::parse_named_value(input, errors)?;
+
+        Ok(value.into())
+    }
 }
 
 impl ParseInputTokens for String {

diff --git a/crates/codegen/language/definition/src/model/item.rs b/crates/codegen/language/definition/src/model/item.rs
@@ -1,3 +1,5 @@
+use std::rc::Rc;
+
 use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, WriteOutputTokens};
 use serde::{Deserialize, Serialize};
 use strum_macros::EnumDiscriminants;
@@ -8,18 +10,18 @@ use crate::model::{
 };
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(EnumDiscriminants, ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, EnumDiscriminants, ParseInputTokens, WriteOutputTokens)]
 pub enum Item {
-    Struct { item: StructItem },
-    Enum { item: EnumItem },
-    Repeated { item: RepeatedItem },
-    Separated { item: SeparatedItem },
-    Precedence { item: PrecedenceItem },
+    Struct { item: Rc<StructItem> },
+    Enum { item: Rc<EnumItem> },
+    Repeated { item: Rc<RepeatedItem> },
+    Separated { item: Rc<SeparatedItem> },
+    Precedence { item: Rc<PrecedenceItem> },
 
-    Trivia { item: TriviaItem },
-    Keyword { item: KeywordItem },
-    Token { item: TokenItem },
-    Fragment { item: FragmentItem },
+    Trivia { item: Rc<TriviaItem> },
+    Keyword { item: Rc<KeywordItem> },
+    Token { item: Rc<TokenItem> },
+    Fragment { item: Rc<FragmentItem> },
 }
 
 impl Item {

diff --git a/crates/codegen/language/definition/src/model/manifest.rs b/crates/codegen/language/definition/src/model/manifest.rs
@@ -1,5 +1,4 @@
 use std::collections::BTreeSet;
-use std::rc::Rc;
 
 use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, WriteOutputTokens};
 use indexmap::IndexSet;
@@ -9,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use crate::model::{Field, Identifier, Item, TriviaParser, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct Language {
     pub name: Identifier,
 
@@ -24,33 +23,33 @@ pub struct Language {
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct Section {
     pub title: String,
     pub topics: Vec<Topic>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct Topic {
     pub title: String,
     pub notes_file: Option<String>,
     pub lexical_context: Option<Identifier>,
 
-    pub items: Vec<Rc<Item>>,
+    pub items: Vec<Item>,
 }
 
 impl Language {
     /// Returns every item in the language definition.
     pub fn items(&self) -> impl Iterator<Item = &Item> {
         self.sections
             .iter()
-            .flat_map(|section| &*section.topics)
-            .flat_map(|topic| topic.items.iter().map(Rc::as_ref))
+            .flat_map(|section| &section.topics)
+            .flat_map(|topic| &topic.items)
     }
 
     /// Returns a flattened iterator over items along with section and topic they belong to.
-    pub fn items_with_section(&self) -> impl Iterator<Item = (&Section, &Topic, &Rc<Item>)> {
+    pub fn items_with_section(&self) -> impl Iterator<Item = (&Section, &Topic, &Item)> {
         self.sections.iter().flat_map(|section| {
             section
                 .topics

diff --git a/crates/codegen/language/definition/src/model/non_terminals/enum_.rs b/crates/codegen/language/definition/src/model/non_terminals/enum_.rs
@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
 use crate::model::{Identifier, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct EnumItem {
     pub name: Identifier,
 
@@ -14,7 +14,7 @@ pub struct EnumItem {
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct EnumVariant {
     pub reference: Identifier,
 

diff --git a/crates/codegen/language/definition/src/model/non_terminals/field.rs b/crates/codegen/language/definition/src/model/non_terminals/field.rs
@@ -4,21 +4,21 @@ use serde::{Deserialize, Serialize};
 use crate::model::{Identifier, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct FieldsErrorRecovery {
     pub terminator: Option<Identifier>,
     pub delimiters: Option<FieldDelimiters>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct FieldDelimiters {
     pub open: Identifier,
     pub close: Identifier,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub enum Field {
     Required {
         reference: Identifier,

diff --git a/crates/codegen/language/definition/src/model/non_terminals/precedence.rs b/crates/codegen/language/definition/src/model/non_terminals/precedence.rs
@@ -1,30 +1,32 @@
+use std::rc::Rc;
+
 use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, WriteOutputTokens};
 use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
 
 use crate::model::{Field, FieldsErrorRecovery, Identifier, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct PrecedenceItem {
     pub name: Identifier,
 
     pub enabled: Option<VersionSpecifier>,
 
-    pub precedence_expressions: Vec<PrecedenceExpression>,
+    pub precedence_expressions: Vec<Rc<PrecedenceExpression>>,
     pub primary_expressions: Vec<PrimaryExpression>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct PrecedenceExpression {
     pub name: Identifier,
 
     pub operators: Vec<PrecedenceOperator>,
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct PrecedenceOperator {
     pub model: OperatorModel,
 
@@ -35,7 +37,7 @@ pub struct PrecedenceOperator {
 }
 
 #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Hash, PartialOrd, Ord, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub enum OperatorModel {
     Prefix,
     Postfix,
@@ -44,7 +46,7 @@ pub enum OperatorModel {
 }
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct PrimaryExpression {
     pub reference: Identifier,
 

diff --git a/crates/codegen/language/definition/src/model/non_terminals/repeated.rs b/crates/codegen/language/definition/src/model/non_terminals/repeated.rs
@@ -4,10 +4,10 @@ use serde::{Deserialize, Serialize};
 use crate::model::{Identifier, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct RepeatedItem {
     pub name: Identifier,
-    pub repeated: Identifier,
+    pub reference: Identifier,
 
     pub enabled: Option<VersionSpecifier>,
 }
diff --git a/crates/codegen/language/definition/src/model/non_terminals/separated.rs b/crates/codegen/language/definition/src/model/non_terminals/separated.rs
@@ -4,10 +4,10 @@ use serde::{Deserialize, Serialize};
 use crate::model::{Identifier, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
-#[derive_spanned_type(ParseInputTokens, WriteOutputTokens)]
+#[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
 pub struct SeparatedItem {
     pub name: Identifier,
-    pub separated: Identifier,
+    pub reference: Identifier,
     pub separator: Identifier,
 
     pub enabled: Option<VersionSpecifier>,