diff --git a/rust/candid_parser/src/bindings/motoko.rs b/rust/candid_parser/src/bindings/motoko.rs index 6302fde39..eafa1150d 100644 --- a/rust/candid_parser/src/bindings/motoko.rs +++ b/rust/candid_parser/src/bindings/motoko.rs @@ -102,7 +102,7 @@ fn pp_ty_rich<'a>(ty: &'a Type, syntax: Option<&'a IDLType>) -> RcDoc<'a> { pp_service(meths, Some(methods)) } (TypeInner::Class(ref args, t), Some(IDLType::ClassT(_, syntax_t))) => { - pp_class((args, t), Some(syntax_t)) + pp_class((args, t), Some(&syntax_t.kind)) } (TypeInner::Record(ref fields), Some(IDLType::RecordT(syntax_fields))) => { pp_record(fields, Some(syntax_fields)) @@ -111,9 +111,11 @@ fn pp_ty_rich<'a>(ty: &'a Type, syntax: Option<&'a IDLType>) -> RcDoc<'a> { pp_variant(fields, Some(syntax_fields)) } (TypeInner::Opt(ref inner), Some(IDLType::OptT(syntax))) => { - str("?").append(pp_ty_rich(inner, Some(syntax))) + str("?").append(pp_ty_rich(inner, Some(&syntax.kind))) + } + (TypeInner::Vec(ref inner), Some(IDLType::VecT(syntax))) => { + pp_vec(inner, Some(&syntax.kind)) } - (TypeInner::Vec(ref inner), Some(IDLType::VecT(syntax))) => pp_vec(inner, Some(syntax)), (_, _) => pp_ty(ty), } } @@ -212,7 +214,7 @@ fn pp_service<'a>(serv: &'a [(String, Type)], syntax: Option<&'a [syntax::Bindin if let Some(bs) = syntax { if let Some(b) = bs.iter().find(|b| &b.id == id) { docs = pp_docs(&b.docs); - syntax_field_ty = Some(&b.typ) + syntax_field_ty = Some(&b.typ.kind) } } docs.append(escape(id, true)) @@ -244,7 +246,7 @@ fn find_field<'a>( if let Some(bs) = fields { if let Some(field) = bs.iter().find(|b| b.label == *label) { docs = pp_docs(&field.docs); - syntax_field_ty = Some(&field.typ); + syntax_field_ty = Some(&field.typ.kind); } }; (docs, syntax_field_ty) @@ -302,7 +304,7 @@ fn pp_defs<'a>(env: &'a TypeEnv, prog: &'a IDLMergedProg) -> RcDoc<'a> { docs.append(kwd("public type")) .append(escape(id, false)) .append(" = ") - .append(pp_ty_rich(ty, syntax.map(|b| &b.typ))) + .append(pp_ty_rich(ty, syntax.map(|b| &b.typ.kind))) .append(";") })) } @@ -311,25 +313,25 @@ fn pp_actor<'a>(ty: &'a Type, syntax: Option<&'a IDLActorType>) -> RcDoc<'a> { let self_doc = kwd("public type Self ="); match ty.as_ref() { TypeInner::Service(ref serv) => match syntax { - Some(IDLActorType { - typ: IDLType::ServT(ref fields), - docs, - }) => { - let docs = pp_docs(docs); - docs.append(self_doc).append(pp_service(serv, Some(fields))) - } - _ => pp_service(serv, None), + Some(IDLActorType { typ, docs, .. }) => match &typ.kind { + IDLType::ServT(fields) => { + let docs = pp_docs(docs); + docs.append(self_doc).append(pp_service(serv, Some(fields))) + } + _ => pp_service(serv, None), + }, + None => pp_service(serv, None), }, TypeInner::Class(ref args, ref t) => match syntax { - Some(IDLActorType { - typ: IDLType::ClassT(_, syntax_t), - docs, - }) => { - let docs = pp_docs(docs); - docs.append(self_doc) - .append(pp_class((args, t), Some(syntax_t))) - } - _ => self_doc.append(pp_class((args, t), None)), + Some(IDLActorType { typ, docs, .. }) => match &typ.kind { + IDLType::ClassT(_, syntax_t) => { + let docs = pp_docs(docs); + docs.append(self_doc) + .append(pp_class((args, t), Some(&syntax_t.kind))) + } + _ => self_doc.append(pp_class((args, t), None)), + }, + None => self_doc.append(pp_class((args, t), None)), }, TypeInner::Var(_) => self_doc.append(pp_ty(ty)), _ => unreachable!(), diff --git a/rust/candid_parser/src/bindings/rust.rs b/rust/candid_parser/src/bindings/rust.rs index ef1439dab..d698b230c 100644 --- a/rust/candid_parser/src/bindings/rust.rs +++ b/rust/candid_parser/src/bindings/rust.rs @@ -150,7 +150,7 @@ fn find_field<'a>( if let Some(bs) = fields { if let Some(field) = bs.iter().find(|b| b.label == *label) { docs = pp_docs(&field.docs); - syntax_field_ty = Some(&field.typ); + syntax_field_ty = Some(&field.typ.kind); } }; (docs, syntax_field_ty) @@ -178,7 +178,7 @@ fn actor_methods(actor: Option<&IDLActorType>) -> &[syntax::Binding] { None => return &[], }; - match typ { + match &typ.kind { IDLType::ServT(methods) => methods, IDLType::ClassT(_, inner) => { if let IDLType::ServT(methods) = inner.as_ref() { @@ -482,7 +482,7 @@ fn test_{test_name}() {{ .unwrap_or_else(|| to_identifier_case(id, IdentifierCase::UpperCamel).0); let syntax = self.prog.lookup(id); let syntax_ty = syntax - .map(|b| &b.typ) + .map(|b| &b.typ.kind) .or_else(|| self.generated_types.get(*id).map(|t| &**t)); let docs = syntax .map(|b| pp_docs(b.docs.as_ref())) @@ -958,8 +958,9 @@ impl<'b> NominalState<'_, 'b> { let elem = StateElem::Label(&lab); let old = self.state.push_state(&elem); path.push(TypePath::RecordField(id.to_string())); - let syntax_field = syntax_fields - .and_then(|s| s.iter().find(|f| f.label == **id).map(|f| &f.typ)); + let syntax_field = syntax_fields.and_then(|s| { + s.iter().find(|f| f.label == **id).map(|f| &f.typ.kind) + }); let ty = self.nominalize(env, path, ty, syntax_field); path.pop(); self.state.pop_state(old, elem); @@ -1003,8 +1004,9 @@ impl<'b> NominalState<'_, 'b> { } else { path.push(TypePath::VariantField(id.to_string())); } - let syntax_field = syntax_fields - .and_then(|s| s.iter().find(|f| f.label == **id).map(|f| &f.typ)); + let syntax_field = syntax_fields.and_then(|s| { + s.iter().find(|f| f.label == **id).map(|f| &f.typ.kind) + }); let ty = self.nominalize(env, path, ty, syntax_field); path.pop(); self.state.pop_state(old, StateElem::Label(&lab)); @@ -1167,7 +1169,7 @@ impl<'b> NominalState<'_, 'b> { for (id, ty) in self.state.env.0.iter() { let elem = StateElem::Label(id); let old = self.state.push_state(&elem); - let syntax = prog.lookup(id).map(|t| &t.typ); + let syntax = prog.lookup(id).map(|t| &t.typ.kind); let ty = self.nominalize(&mut res, &mut vec![TypePath::Id(id.clone())], ty, syntax); res.0.insert(id.to_string(), ty); self.state.pop_state(old, elem); diff --git a/rust/candid_parser/src/bindings/typescript.rs b/rust/candid_parser/src/bindings/typescript.rs index ce09e3754..8284cebfe 100644 --- a/rust/candid_parser/src/bindings/typescript.rs +++ b/rust/candid_parser/src/bindings/typescript.rs @@ -17,7 +17,7 @@ fn find_field<'a>( if let Some(bs) = fields { if let Some(field) = bs.iter().find(|b| b.label == *label) { docs = pp_docs(&field.docs); - syntax_field_ty = Some(&field.typ); + syntax_field_ty = Some(&field.typ.kind); } }; (docs, syntax_field_ty) @@ -40,10 +40,10 @@ fn pp_ty_rich<'a>( pp_service(env, serv, Some(syntax_serv)) } (TypeInner::Opt(ref t), Some(IDLType::OptT(syntax_inner))) => { - pp_opt(env, t, Some(syntax_inner), is_ref) + pp_opt(env, t, Some(&syntax_inner.kind), is_ref) } (TypeInner::Vec(ref t), Some(IDLType::VecT(syntax_inner))) => { - pp_vec(env, t, Some(syntax_inner), is_ref) + pp_vec(env, t, Some(&syntax_inner.kind), is_ref) } (_, _) => pp_ty(env, ty, is_ref), } @@ -266,7 +266,7 @@ fn pp_defs<'a>(env: &'a TypeEnv, def_list: &'a [&'a str], prog: &'a IDLMergedPro lines(def_list.iter().map(|id| { let ty = env.find_type(id).unwrap(); let syntax = prog.lookup(id); - let syntax_ty = syntax.map(|s| &s.typ); + let syntax_ty = syntax.map(|s| &s.typ.kind); let docs = syntax .map(|b| pp_docs(b.docs.as_ref())) .unwrap_or(RcDoc::nil()); @@ -309,7 +309,7 @@ fn pp_actor<'a>(env: &'a TypeEnv, ty: &'a Type, syntax: Option<&'a IDLType>) -> .append(str(" {}")), TypeInner::Class(_, t) => { if let Some(IDLType::ClassT(_, syntax_t)) = syntax { - pp_actor(env, t, Some(syntax_t)) + pp_actor(env, t, Some(&syntax_t.kind)) } else { pp_actor(env, t, None) } @@ -333,11 +333,15 @@ import type { IDL } from '@dfinity/candid'; .as_ref() .map(|s| pp_docs(s.docs.as_ref())) .unwrap_or(RcDoc::nil()); - docs.append(pp_actor(env, actor, syntax_actor.as_ref().map(|s| &s.typ))) - .append(RcDoc::line()) - .append("export declare const idlFactory: IDL.InterfaceFactory;") - .append(RcDoc::line()) - .append("export declare const init: (args: { IDL: typeof IDL }) => IDL.Type[];") + docs.append(pp_actor( + env, + actor, + syntax_actor.as_ref().map(|s| &s.typ.kind), + )) + .append(RcDoc::line()) + .append("export declare const idlFactory: IDL.InterfaceFactory;") + .append(RcDoc::line()) + .append("export declare const init: (args: { IDL: typeof IDL }) => IDL.Type[];") } }; let doc = RcDoc::text(header) diff --git a/rust/candid_parser/src/grammar.lalrpop b/rust/candid_parser/src/grammar.lalrpop index 584be1729..0987780dc 100644 --- a/rust/candid_parser/src/grammar.lalrpop +++ b/rust/candid_parser/src/grammar.lalrpop @@ -1,7 +1,11 @@ use super::test::{Assert, Input, Test}; -use super::token::{Token, error2, LexicalError, Span, TriviaMap}; +use super::token::{error2, LexicalError, ParserError, Span, Token, TriviaMap}; use candid::{Principal, types::Label}; -use crate::syntax::{IDLType, PrimType, TypeField, FuncType, Binding, Dec, IDLProg, IDLTypes, IDLInitArgs, IDLActorType}; +use crate::syntax::{ + Binding, Dec, FuncType, IDLActorType, IDLInitArgs, IDLProg, IDLType, IDLTypeWithSpan, IDLTypes, + TypeField, +}; +use crate::syntax::PrimType; use candid::types::value::{IDLField, IDLValue, IDLArgs, VariantValue}; use candid::types::{TypeEnv, FuncMode}; use candid::utils::check_unique; @@ -113,7 +117,7 @@ AnnVal: IDLValue = { => <>, > ":" > =>? { let env = TypeEnv::new(); - let typ = crate::typing::ast_to_type(&env, &typ.0).map_err(|e| error2(e, typ.1))?; + let typ = crate::typing::ast_to_type(&env, &typ.0.kind).map_err(|e| error2(e, typ.1))?; arg.0.annotate_type(true, &env, &typ).map_err(|e| error2(e, arg.1)) } } @@ -161,70 +165,113 @@ RecordField: IDLField = { // Type pub Typs: IDLTypes = TupTyp => IDLTypes { args:<> }; -pub Typ: IDLType = { +pub Typ: IDLTypeWithSpan = { PrimTyp => <>, - "opt" => IDLType::OptT(Box::new(<>)), - "vec" => IDLType::VecT(Box::new(<>)), - "blob" => IDLType::VecT(Box::new(IDLType::PrimT(PrimType::Nat8))), - "record" "{" >> "}" =>? { + "opt" => IDLTypeWithSpan::new(l..r, IDLType::OptT(Box::new(typ))), + "vec" => IDLTypeWithSpan::new(l..r, IDLType::VecT(Box::new(typ))), + "blob" => { + let span = l..r; + let elem = IDLTypeWithSpan::new(span.clone(), IDLType::PrimT(PrimType::Nat8)); + IDLTypeWithSpan::new(span, IDLType::VecT(Box::new(elem))) + }, + "record" "{" >> "}" =>? { let mut id: u32 = 0; - let span = <>.1.clone(); - let mut fs: Vec = <>.0.iter().map(|f| { - let label = match f.label { - Label::Unnamed(_) => { id = id + 1; Label::Unnamed(id - 1) }, - ref l => { id = l.get_id() + 1; l.clone() }, - }; - TypeField { label, typ: f.typ.clone(), docs: f.docs.clone() } + let span = l..r; + let inner_span = fields.1.clone(); + let mut fs: Vec = fields.0.iter().map(|f| { + let label = match &f.label { + Label::Unnamed(_) => { + id += 1; + Label::Unnamed(id - 1) + } + ref l => { + id = l.get_id() + 1; + (*l).clone() + } + }; + TypeField { + label, + typ: f.typ.clone(), + docs: f.docs.clone(), + span: f.span.clone(), + } }).collect(); fs.sort_unstable_by_key(|TypeField { label, .. }| label.get_id()); - check_unique(fs.iter().map(|f| &f.label)).map_err(|e| error2(e, span))?; - Ok(IDLType::RecordT(fs)) + check_unique(fs.iter().map(|f| &f.label)).map_err(|e| error2(e, inner_span))?; + Ok(IDLTypeWithSpan::new(span, IDLType::RecordT(fs))) }, - "variant" "{" >> "}" =>? { - let span = fs.1.clone(); - fs.0.sort_unstable_by_key(|TypeField { label, .. }| label.get_id()); - check_unique(fs.0.iter().map(|f| &f.label)).map_err(|e| error2(e, span))?; - Ok(IDLType::VariantT(fs.0)) + "variant" "{" >> "}" =>? { + let span = l..r; + let inner_span = fields.1.clone(); + fields.0.sort_unstable_by_key(|TypeField { label, .. }| label.get_id()); + check_unique(fields.0.iter().map(|f| &f.label)).map_err(|e| error2(e, inner_span))?; + Ok(IDLTypeWithSpan::new(span, IDLType::VariantT(fields.0))) }, - "func" => IDLType::FuncT(<>), - "service" => IDLType::ServT(<>), - "principal" => IDLType::PrincipalT, -} - -PrimTyp: IDLType = { - "null" => IDLType::PrimT(PrimType::Null), - "id" => { - match PrimType::str_to_enum(&<>) { - Some(p) => IDLType::PrimT(p), - None => IDLType::VarT(<>), - } + "func" => IDLTypeWithSpan::new(l..r, IDLType::FuncT(typ)), + "service" => IDLTypeWithSpan::new(l..r, IDLType::ServT(typ)), + "principal" => IDLTypeWithSpan::new(l..r, IDLType::PrincipalT), +} + +PrimTyp: IDLTypeWithSpan = { + "null" => IDLTypeWithSpan::new(l..r, IDLType::PrimT(PrimType::Null)), + => { + let span = l..r; + match PrimType::str_to_enum(&ident) { + Some(p) => IDLTypeWithSpan::new(span, IDLType::PrimT(p)), + None => IDLTypeWithSpan::new(span, IDLType::VarT(ident)), + } }, } FieldTyp: TypeField = { - ":" =>? Ok(TypeField { label: Label::Id(id), typ, docs: doc_comment.unwrap_or_default() }), - ":" => TypeField { label: Label::Named(n), typ, docs: doc_comment.unwrap_or_default() }, + ":" =>? Ok(TypeField { + label: Label::Id(id), + typ, + docs: doc_comment.unwrap_or_default(), + span: l..r, + }), + ":" => TypeField { + label: Label::Named(n), + typ, + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, } RecordFieldTyp: TypeField = { FieldTyp => <>, - => TypeField { label: Label::Unnamed(0), typ, docs: doc_comment.unwrap_or_default() }, + => TypeField { + label: Label::Unnamed(0), + typ, + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, } VariantFieldTyp: TypeField = { FieldTyp => <>, - => TypeField { label: Label::Named(n), typ: IDLType::PrimT(PrimType::Null), docs: doc_comment.unwrap_or_default() }, - =>? Ok(TypeField { label: Label::Id(id), typ: IDLType::PrimT(PrimType::Null), docs: doc_comment.unwrap_or_default() }), + => TypeField { + label: Label::Named(n), + typ: IDLTypeWithSpan::new(l..r, IDLType::PrimT(PrimType::Null)), + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, + =>? Ok(TypeField { + label: Label::Id(id), + typ: IDLTypeWithSpan::new(l..r, IDLType::PrimT(PrimType::Null)), + docs: doc_comment.unwrap_or_default(), + span: l..r, + }), } -TupTyp: Vec = "(" > ")" => <>; +TupTyp: Vec = "(" > ")" => <>; FuncTyp: FuncType = { "->" => FuncType { modes, args, rets }, } -ArgTyp: IDLType = { +ArgTyp: IDLTypeWithSpan = { Typ => <>, Name ":" => <>, } @@ -245,33 +292,97 @@ ActorTyp: Vec = { } MethTyp: Binding = { - ":" => Binding { id: n, typ: IDLType::FuncT(f), docs: doc_comment.unwrap_or_default() }, - ":" => Binding { id: n, typ: IDLType::VarT(id), docs: doc_comment.unwrap_or_default() }, + ":" => Binding { + id: n, + typ: IDLTypeWithSpan::new(l..r, IDLType::FuncT(f)), + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, + ":" => Binding { + id: n, + typ: IDLTypeWithSpan::new(l..r, IDLType::VarT(id)), + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, } // Type declarations Def: Dec = { - "type" "=" => Dec::TypD(Binding { id: id, typ: t, docs: doc_comment.unwrap_or_default() }), - "import" => Dec::ImportType(<>), - "import" "service" => Dec::ImportServ(<>), + "type" "=" => Dec::TypD(Binding { + id: id, + typ: t, + docs: doc_comment.unwrap_or_default(), + span: l..r, + }), + "import" => Dec::ImportType { path: text, span: l..r }, + "import" "service" => Dec::ImportServ { path: text, span: l..r }, +} + +RecoverDef: Result = { + Def => Ok(<>), + => Err(err.error), } -Actor: IDLType = { - ActorTyp => IDLType::ServT(<>), - "id" => IDLType::VarT(<>), +Actor: IDLTypeWithSpan = { + => IDLTypeWithSpan::new(l..r, IDLType::ServT(typ)), + => IDLTypeWithSpan::new(l..r, IDLType::VarT(name)), } MainActor: IDLActorType = { - "service" "id"? ":" ";"? => IDLActorType { typ: t, docs: doc_comment.unwrap_or_default() }, - "service" "id"? ":" "->" ";"? => IDLActorType { typ: IDLType::ClassT(args, Box::new(t)), docs: doc_comment.unwrap_or_default() }, + "service" "id"? ":" ";"? => IDLActorType { + typ: t, + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, + "service" "id"? ":" "->" ";"? => IDLActorType { + typ: IDLTypeWithSpan::new(l..r, IDLType::ClassT(args, Box::new(t))), + docs: doc_comment.unwrap_or_default(), + span: l..r, + }, +} + +MainActorLossy: Result = { + MainActor => Ok(<>), + <_doc_comment: DocComment> "service" "id"? ":" ";"? => Err(err.error), + <_doc_comment: DocComment> "service" "id"? ":" <_args:TupTyp> "->" ";"? => Err(err.error), + <_doc_comment: DocComment> "service" => Err(err.error), } pub IDLProg: IDLProg = { - > => IDLProg { decs, actor } + > => IDLProg { decs, actor, span: l..r } +} + +pub IDLProgLossy: (Option, Vec) = { + > => { + let mut errors = Vec::new(); + let mut ok_decs = Vec::new(); + for dec in decs { + match dec { + Ok(dec) => ok_decs.push(dec), + Err(err) => errors.push(err), + } + } + let actor = match actor { + None => None, + Some(result) => match result { + Ok(actor) => Some(actor), + Err(err) => { + errors.push(err); + None + } + }, + }; + let prog = if ok_decs.is_empty() && actor.is_none() { + None + } else { + Some(IDLProg { decs: ok_decs, actor, span: l..r }) + }; + (prog, errors) + } } pub IDLInitArgs: IDLInitArgs = { - > => IDLInitArgs { decs, args } + > => IDLInitArgs { decs, args, span: l..r } } // Test file. Follows the "specification" in test/README.md @@ -288,14 +399,25 @@ Assert: Assert = > =>? { }; Assertion: Assert = { - ":" => Assert { left, right: None, typ, pass: true, desc }, - "!:" => Assert { left, right: None, typ, pass: false, desc }, - "==" ":" => Assert { left, right: Some(right), typ, pass: true, desc }, - "!=" ":" => Assert { left, right: Some(right), typ, pass: false, desc }, + ":" => { + Assert { left, right: None, typ, pass: true, desc } + }, + "!:" => { + Assert { left, right: None, typ, pass: false, desc } + }, + "==" ":" => { + Assert { left, right: Some(right), typ, pass: true, desc } + }, + "!=" ":" => { + Assert { left, right: Some(right), typ, pass: false, desc } + }, } pub Test: Test = { - > > => Test { defs, asserts }, + > > => { + let defs = defs.into_iter().map(Into::into).collect(); + Test { defs, asserts } + }, } // Common util diff --git a/rust/candid_parser/src/lib.rs b/rust/candid_parser/src/lib.rs index 63d3674b4..ec90ed651 100644 --- a/rust/candid_parser/src/lib.rs +++ b/rust/candid_parser/src/lib.rs @@ -151,3 +151,28 @@ pub fn parse_idl_value(s: &str) -> crate::Result { let lexer = token::Tokenizer::new(s); Ok(grammar::ArgParser::new().parse(None, lexer)?) } + +/// Parse an `IDLProg` from any iterator over lexer tokens. +/// +/// This allows callers to provide their own iterator that may, for example, +/// record every consumed token before it is fed to the parser. +pub fn parse_idl_prog_from_tokens( + trivia: Option<&token::TriviaMap>, + tokens: I, +) -> std::result::Result +where + I: IntoIterator>, +{ + IDLProg::parse_from_tokens(trivia, tokens) +} + +/// Parse a `.did` program while attempting to recover from syntax errors. +/// +/// The returned vector contains every `lalrpop_util::ParseError` that was reported +/// during parsing. Lexer errors bubble up as `ParseError::User { .. }`, which +/// lets callers distinguish them from structural syntax issues. +pub fn parse_prog_lossy(s: &str) -> (Option, Vec) { + let trivia = token::TriviaMap::default(); + let lexer = token::Tokenizer::new_with_trivia(s, trivia.clone()); + IDLProg::parse_lossy_from_tokens(Some(&trivia), lexer) +} diff --git a/rust/candid_parser/src/syntax/mod.rs b/rust/candid_parser/src/syntax/mod.rs index 8a9c51f33..7cce01dbe 100644 --- a/rust/candid_parser/src/syntax/mod.rs +++ b/rust/candid_parser/src/syntax/mod.rs @@ -2,7 +2,10 @@ mod pretty; pub use pretty::pretty_print; -use crate::error; +use crate::{ + error, + token::{LexicalError, ParserError, Span, Token, TriviaMap}, +}; use anyhow::{anyhow, bail, Context, Result}; use candid::{ idl_hash, @@ -10,17 +13,54 @@ use candid::{ }; use std::collections::HashMap; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Spanned { + pub value: T, + pub span: Span, +} + +impl Spanned { + pub fn new(value: T, span: Span) -> Self { + Spanned { value, span } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IDLTypeWithSpan { + pub span: Span, + pub kind: IDLType, +} + +impl IDLTypeWithSpan { + pub fn new(span: Span, kind: IDLType) -> Self { + IDLTypeWithSpan { span, kind } + } + + pub fn synthetic(kind: IDLType) -> Self { + IDLTypeWithSpan { span: 0..0, kind } + } +} + +impl std::str::FromStr for IDLTypeWithSpan { + type Err = error::Error; + fn from_str(str: &str) -> error::Result { + let trivia = super::token::TriviaMap::default(); + let lexer = super::token::Tokenizer::new_with_trivia(str, trivia.clone()); + Ok(super::grammar::TypParser::new().parse(Some(&trivia), lexer)?) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum IDLType { PrimT(PrimType), VarT(String), FuncT(FuncType), - OptT(Box), - VecT(Box), + OptT(Box), + VecT(Box), RecordT(Vec), VariantT(Vec), ServT(Vec), - ClassT(Vec, Box), + ClassT(Vec, Box), PrincipalT, } @@ -40,18 +80,40 @@ impl IDLType { } } +impl From for IDLType { + fn from(value: IDLTypeWithSpan) -> Self { + value.kind + } +} + +impl AsRef for IDLTypeWithSpan { + fn as_ref(&self) -> &IDLType { + &self.kind + } +} + +impl AsRef for Box { + fn as_ref(&self) -> &IDLType { + &self.kind + } +} + +impl AsRef for IDLType { + fn as_ref(&self) -> &IDLType { + self + } +} + impl std::str::FromStr for IDLType { type Err = error::Error; fn from_str(str: &str) -> error::Result { - let trivia = super::token::TriviaMap::default(); - let lexer = super::token::Tokenizer::new_with_trivia(str, trivia.clone()); - Ok(super::grammar::TypParser::new().parse(Some(&trivia), lexer)?) + Ok(IDLTypeWithSpan::from_str(str)?.kind) } } #[derive(Debug, Clone)] pub struct IDLTypes { - pub args: Vec, + pub args: Vec, } impl std::str::FromStr for IDLTypes { @@ -106,41 +168,45 @@ pub enum PrimType { #[derive(Debug, Clone, PartialEq, Eq)] pub struct FuncType { pub modes: Vec, - pub args: Vec, - pub rets: Vec, + pub args: Vec, + pub rets: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct TypeField { pub label: Label, - pub typ: IDLType, + pub typ: IDLTypeWithSpan, pub docs: Vec, + pub span: Span, } #[derive(Debug)] pub enum Dec { TypD(Binding), - ImportType(String), - ImportServ(String), + ImportType { path: String, span: Span }, + ImportServ { path: String, span: Span }, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct Binding { pub id: String, - pub typ: IDLType, + pub typ: IDLTypeWithSpan, pub docs: Vec, + pub span: Span, } #[derive(Debug, Clone)] pub struct IDLActorType { - pub typ: IDLType, + pub typ: IDLTypeWithSpan, pub docs: Vec, + pub span: Span, } #[derive(Debug)] pub struct IDLProg { pub decs: Vec, pub actor: Option, + pub span: Span, } impl IDLProg { @@ -153,6 +219,29 @@ impl IDLProg { } }) } + + pub fn parse_from_tokens( + trivia: Option<&TriviaMap>, + tokens: I, + ) -> std::result::Result + where + I: IntoIterator>, + { + super::grammar::IDLProgParser::new().parse(trivia, tokens) + } + + pub fn parse_lossy_from_tokens( + trivia: Option<&TriviaMap>, + tokens: I, + ) -> (Option, Vec) + where + I: IntoIterator>, + { + match super::grammar::IDLProgLossyParser::new().parse(trivia, tokens) { + Ok(result) => result, + Err(err) => (None, vec![err]), + } + } } impl std::str::FromStr for IDLProg { @@ -160,14 +249,15 @@ impl std::str::FromStr for IDLProg { fn from_str(str: &str) -> error::Result { let trivia = super::token::TriviaMap::default(); let lexer = super::token::Tokenizer::new_with_trivia(str, trivia.clone()); - Ok(super::grammar::IDLProgParser::new().parse(Some(&trivia), lexer)?) + Ok(Self::parse_from_tokens(Some(&trivia), lexer)?) } } #[derive(Debug)] pub struct IDLInitArgs { pub decs: Vec, - pub args: Vec, + pub args: Vec, + pub span: Span, } impl std::str::FromStr for IDLInitArgs { @@ -219,32 +309,35 @@ impl IDLMergedProg { } pub fn resolve_actor(&self) -> Result> { - let (init_args, top_level_docs, mut methods) = match &self.main_actor { + let mut init_args: Option> = None; + let mut top_level_docs: Vec = vec![]; + let mut actor_span: Span = 0..0; + let mut methods = match &self.main_actor { None => { if self.service_imports.is_empty() { return Ok(None); - } else { - (None, vec![], vec![]) + } + vec![] + } + Some(actor) if self.service_imports.is_empty() => return Ok(Some(actor.clone())), + Some(actor) => { + top_level_docs = actor.docs.clone(); + actor_span = actor.span.clone(); + match &actor.typ.kind { + IDLType::ClassT(args, inner) => { + init_args = Some(args.clone()); + self.chase_service((**inner).clone(), None)? + } + _ => self.chase_service(actor.typ.clone(), None)?, } } - Some(t) if self.service_imports.is_empty() => return Ok(Some(t.clone())), - Some(IDLActorType { - typ: IDLType::ClassT(args, inner), - docs, - }) => ( - Some(args.clone()), - docs.clone(), - self.chase_service(*inner.clone(), None)?, - ), - Some(ty) => ( - None, - ty.docs.clone(), - self.chase_service(ty.typ.clone(), None)?, - ), }; for (name, typ) in &self.service_imports { methods.extend(self.chase_service(typ.typ.clone(), Some(name))?); + if top_level_docs.is_empty() { + top_level_docs = typ.docs.clone(); + } } let mut hashes: HashMap = HashMap::new(); @@ -255,20 +348,27 @@ impl IDLMergedProg { } } + let service_type = IDLTypeWithSpan::synthetic(IDLType::ServT(methods)); let typ = if let Some(args) = init_args { - IDLType::ClassT(args, Box::new(IDLType::ServT(methods))) + IDLTypeWithSpan::synthetic(IDLType::ClassT(args, Box::new(service_type.clone()))) } else { - IDLType::ServT(methods) + service_type }; + Ok(Some(IDLActorType { typ, docs: top_level_docs, + span: actor_span, })) } // NOTE: We don't worry about cyclic type definitions, as we rule those out earlier when checking the type decs - fn chase_service(&self, ty: IDLType, import_name: Option<&str>) -> Result> { - match ty { + fn chase_service( + &self, + ty: IDLTypeWithSpan, + import_name: Option<&str>, + ) -> Result> { + match ty.kind { IDLType::VarT(v) => { let resolved = self .typ_decs @@ -278,9 +378,9 @@ impl IDLMergedProg { self.chase_service(resolved.typ.clone(), import_name) } IDLType::ServT(bindings) => Ok(bindings), - ty => Err(import_name + ty_kind => Err(import_name .map(|name| anyhow!("Imported service file \"{name}\" has a service constructor")) - .unwrap_or(anyhow!("not a service type: {:?}", ty))), + .unwrap_or(anyhow!("not a service type: {:?}", ty_kind))), } } } diff --git a/rust/candid_parser/src/syntax/pretty.rs b/rust/candid_parser/src/syntax/pretty.rs index 1b1b1970e..444b103b3 100644 --- a/rust/candid_parser/src/syntax/pretty.rs +++ b/rust/candid_parser/src/syntax/pretty.rs @@ -5,7 +5,10 @@ use crate::{ candid::{pp_docs, pp_label_raw, pp_modes, pp_text}, utils::{concat, enclose, enclose_space, ident, kwd, lines, str, INDENT_SPACE, LINE_WIDTH}, }, - syntax::{Binding, FuncType, IDLActorType, IDLMergedProg, IDLType, PrimType, TypeField}, + syntax::{ + Binding, FuncType, IDLActorType, IDLMergedProg, IDLType, IDLTypeWithSpan, PrimType, + TypeField, + }, }; fn pp_ty(ty: &IDLType) -> RcDoc<'_> { @@ -42,10 +45,10 @@ fn pp_ty(ty: &IDLType) -> RcDoc<'_> { fn pp_field(field: &TypeField, is_variant: bool) -> RcDoc<'_> { let docs = pp_docs(&field.docs); - let ty_doc = if is_variant && field.typ == IDLType::PrimT(PrimType::Null) { + let ty_doc = if is_variant && matches!(field.typ.kind, IDLType::PrimT(PrimType::Null)) { RcDoc::nil() } else { - kwd(" :").append(pp_ty(&field.typ)) + kwd(" :").append(pp_ty(&field.typ.kind)) }; docs.append(pp_label_raw(&field.label)).append(ty_doc) } @@ -55,21 +58,21 @@ fn pp_fields(fs: &[TypeField], is_variant: bool) -> RcDoc<'_> { enclose_space("{", concat(fields, ";"), "}") } -fn pp_opt(ty: &IDLType) -> RcDoc<'_> { - kwd("opt").append(pp_ty(ty)) +fn pp_opt(ty: &IDLTypeWithSpan) -> RcDoc<'_> { + kwd("opt").append(pp_ty(&ty.kind)) } -fn pp_vec(ty: &IDLType) -> RcDoc<'_> { - if matches!(ty, IDLType::PrimT(PrimType::Nat8)) { +fn pp_vec(ty: &IDLTypeWithSpan) -> RcDoc<'_> { + if matches!(ty.kind, IDLType::PrimT(PrimType::Nat8)) { str("blob") } else { - kwd("vec").append(pp_ty(ty)) + kwd("vec").append(pp_ty(&ty.kind)) } } fn pp_record(fs: &[TypeField], is_tuple: bool) -> RcDoc<'_> { if is_tuple { - let tuple = concat(fs.iter().map(|f| pp_ty(&f.typ)), ";"); + let tuple = concat(fs.iter().map(|f| pp_ty(&f.typ.kind)), ";"); kwd("record").append(enclose_space("{", tuple, "}")) } else { kwd("record").append(pp_fields(fs, false)) @@ -94,12 +97,12 @@ fn pp_method(func: &FuncType) -> RcDoc<'_> { .nest(INDENT_SPACE) } -fn pp_args(args: &[IDLType]) -> RcDoc<'_> { - let doc = concat(args.iter().map(pp_ty), ","); +fn pp_args(args: &[IDLTypeWithSpan]) -> RcDoc<'_> { + let doc = concat(args.iter().map(|ty| pp_ty(&ty.kind)), ","); enclose("(", doc, ")") } -fn pp_rets(rets: &[IDLType]) -> RcDoc<'_> { +fn pp_rets(rets: &[IDLTypeWithSpan]) -> RcDoc<'_> { pp_args(rets) } @@ -110,9 +113,9 @@ fn pp_service(methods: &[Binding]) -> RcDoc<'_> { fn pp_service_methods(methods: &[Binding]) -> RcDoc<'_> { let methods = methods.iter().map(|b| { let docs = pp_docs(&b.docs); - let func_doc = match b.typ { + let func_doc = match &b.typ.kind { IDLType::FuncT(ref f) => pp_method(f), - IDLType::VarT(_) => pp_ty(&b.typ), + IDLType::VarT(_) => pp_ty(&b.typ.kind), _ => unreachable!(), }; docs.append(pp_text(&b.id)) @@ -123,9 +126,9 @@ fn pp_service_methods(methods: &[Binding]) -> RcDoc<'_> { enclose_space("{", doc, "}") } -fn pp_class<'a>(args: &'a [IDLType], t: &'a IDLType) -> RcDoc<'a> { +fn pp_class<'a>(args: &'a [IDLTypeWithSpan], t: &'a IDLTypeWithSpan) -> RcDoc<'a> { let doc = pp_args(args).append(" ->").append(RcDoc::space()); - match t { + match &t.kind { IDLType::ServT(ref serv) => doc.append(pp_service_methods(serv)), IDLType::VarT(ref s) => doc.append(s), _ => unreachable!(), @@ -138,16 +141,16 @@ fn pp_defs(prog: &IDLMergedProg) -> RcDoc<'_> { docs.append(kwd("type")) .append(ident(&b.id)) .append(kwd("=")) - .append(pp_ty(&b.typ)) + .append(pp_ty(&b.typ.kind)) .append(";") })) } fn pp_actor(actor: &IDLActorType) -> RcDoc<'_> { let docs = pp_docs(&actor.docs); - let service_doc = match actor.typ { + let service_doc = match &actor.typ.kind { IDLType::ServT(ref serv) => pp_service_methods(serv), - IDLType::VarT(_) | IDLType::ClassT(_, _) => pp_ty(&actor.typ), + IDLType::VarT(_) | IDLType::ClassT(_, _) => pp_ty(&actor.typ.kind), _ => unreachable!(), }; docs.append(kwd("service :")).append(service_doc) diff --git a/rust/candid_parser/src/test.rs b/rust/candid_parser/src/test.rs index 10735e941..5ee3bd1fa 100644 --- a/rust/candid_parser/src/test.rs +++ b/rust/candid_parser/src/test.rs @@ -1,5 +1,5 @@ use super::typing::check_prog; -use crate::syntax::{Dec, IDLProg, IDLType}; +use crate::syntax::{Dec, IDLProg, IDLTypeWithSpan}; use crate::{Error, Result}; use candid::types::value::IDLArgs; use candid::types::{Type, TypeEnv}; @@ -7,7 +7,7 @@ use candid::DecoderConfig; const DECODING_COST: usize = 20_000_000; -type TupType = Vec; +type TupType = Vec; pub struct Test { pub defs: Vec, @@ -151,6 +151,7 @@ pub fn check(test: Test) -> Result<()> { let prog = IDLProg { decs: test.defs, actor: None, + span: 0..0, }; check_prog(&mut env, &prog)?; let mut count = 0; @@ -158,7 +159,7 @@ pub fn check(test: Test) -> Result<()> { print!("Checking {} {}...", i + 1, assert.desc()); let mut types = Vec::new(); for ty in assert.typ.iter() { - types.push(super::typing::ast_to_type(&env, ty)?); + types.push(super::typing::ast_to_type(&env, ty.as_ref())?); } let input = assert.left.parse(&env, &types); let pass = if let Some(assert_right) = &assert.right { diff --git a/rust/candid_parser/src/token.rs b/rust/candid_parser/src/token.rs index 37ea787fa..b48c7cb47 100644 --- a/rust/candid_parser/src/token.rs +++ b/rust/candid_parser/src/token.rs @@ -223,7 +223,7 @@ impl LexicalError { } } -pub(crate) type ParserError = ParseError; +pub type ParserError = ParseError; pub fn error(err: E) -> ParserError { ParseError::User { error: LexicalError::new(err, 0..0), diff --git a/rust/candid_parser/src/typing.rs b/rust/candid_parser/src/typing.rs index f661dc8ea..1d6f80747 100644 --- a/rust/candid_parser/src/typing.rs +++ b/rust/candid_parser/src/typing.rs @@ -47,7 +47,11 @@ fn check_prim(prim: &PrimType) -> Type { .into() } -pub fn check_type(env: &Env, t: &IDLType) -> Result { +pub fn check_type(env: &Env, t: T) -> Result +where + T: AsRef, +{ + let t = t.as_ref(); match t { IDLType::PrimT(prim) => Ok(check_prim(prim)), IDLType::VarT(id) => { @@ -137,11 +141,16 @@ fn check_meths(env: &Env, ms: &[Binding]) -> Result> { fn check_defs(env: &mut Env, decs: &[Dec]) -> Result<()> { for dec in decs.iter() { match dec { - Dec::TypD(Binding { id, typ, docs: _ }) => { + Dec::TypD(Binding { + id, + typ, + docs: _, + span: _, + }) => { let t = check_type(env, typ)?; env.te.0.insert(id.to_string(), t); } - Dec::ImportType(_) | Dec::ImportServ(_) => (), + Dec::ImportType { .. } | Dec::ImportServ { .. } => (), } } Ok(()) @@ -190,20 +199,22 @@ fn check_decs(env: &mut Env, decs: &[Dec]) -> Result<()> { fn check_actor(env: &Env, actor: &Option) -> Result> { match actor.as_ref().map(|a| &a.typ) { None => Ok(None), - Some(IDLType::ClassT(ts, t)) => { - let mut args = Vec::new(); - for arg in ts.iter() { - args.push(check_type(env, arg)?); + Some(typ) => match &typ.kind { + IDLType::ClassT(ts, t) => { + let mut args = Vec::new(); + for arg in ts.iter() { + args.push(check_type(env, arg)?); + } + let serv = check_type(env, t)?; + env.te.as_service(&serv)?; + Ok(Some(TypeInner::Class(args, serv).into())) } - let serv = check_type(env, t)?; - env.te.as_service(&serv)?; - Ok(Some(TypeInner::Class(args, serv).into())) - } - Some(typ) => { - let t = check_type(env, typ)?; - env.te.as_service(&t)?; - Ok(Some(t)) - } + _ => { + let t = check_type(env, typ)?; + env.te.as_service(&t)?; + Ok(Some(t)) + } + }, } } @@ -225,8 +236,8 @@ fn load_imports( list: &mut Vec<(PathBuf, String, IDLProg)>, ) -> Result<()> { for dec in prog.decs.iter() { - let include_serv = matches!(dec, Dec::ImportServ(_)); - if let Dec::ImportType(file) | Dec::ImportServ(file) = dec { + let include_serv = matches!(dec, Dec::ImportServ { .. }); + if let Dec::ImportType { path: file, .. } | Dec::ImportServ { path: file, .. } = dec { let path = resolve_path(base, file); match visited.get_mut(&path) { Some(x) => *x = *x || include_serv, diff --git a/rust/candid_parser/tests/parse_prog.rs b/rust/candid_parser/tests/parse_prog.rs new file mode 100644 index 000000000..f61bda8a6 --- /dev/null +++ b/rust/candid_parser/tests/parse_prog.rs @@ -0,0 +1,70 @@ +use candid_parser::{ + parse_idl_prog_from_tokens, parse_prog_lossy, + syntax::Dec, + token::{self, TriviaMap}, +}; + +fn decl_names(decs: &[Dec]) -> Vec<&str> { + decs.iter() + .filter_map(|dec| match dec { + Dec::TypD(binding) => Some(binding.id.as_str()), + _ => None, + }) + .collect() +} + +#[test] +fn parses_program_from_recorded_tokens() { + let source = r#" + // doc: Foo + type Foo = record { field : int }; + service : { foo : () -> (); }; + "#; + let trivia = TriviaMap::default(); + let tokens: Vec<_> = token::Tokenizer::new_with_trivia(source, trivia.clone()).collect(); + let prog = parse_idl_prog_from_tokens(Some(&trivia), tokens).expect("parses"); + + let names = decl_names(&prog.decs); + assert_eq!(names, vec!["Foo"]); + let docs = match &prog.decs[0] { + Dec::TypD(binding) => binding.docs.clone(), + _ => unreachable!(), + }; + assert_eq!(docs, vec!["doc: Foo".to_string()]); + assert!( + prog.actor.is_some(), + "service definition should be preserved" + ); +} + +#[test] +fn lossy_parser_recovers_declarations_and_reports_errors() { + let source = r#" + type Good = record { foo : int }; + type Broken = record { foo : }; + type AlsoGood = record { bar : text }; + service : { + bad : ( -> ) -> (); + ok : () -> (); + }; + "#; + + let (maybe_prog, errors) = parse_prog_lossy(source); + assert!( + errors.len() >= 2, + "should surface multiple parse errors, got {errors:?}" + ); + assert!( + errors + .iter() + .any(|err| matches!(err, lalrpop_util::ParseError::UnrecognizedToken { .. })), + "expected at least one syntax error in the error list" + ); + let prog = maybe_prog.expect("should recover valid declarations"); + let names = decl_names(&prog.decs); + assert_eq!(names, vec!["Good", "AlsoGood"]); + assert!( + prog.actor.is_none(), + "actor should be dropped due to unrecoverable service parse error" + ); +} diff --git a/rust/candid_parser/tests/parse_type.rs b/rust/candid_parser/tests/parse_type.rs index 4bf218ce7..de146e10d 100644 --- a/rust/candid_parser/tests/parse_type.rs +++ b/rust/candid_parser/tests/parse_type.rs @@ -42,7 +42,7 @@ service server : { let actor = ast.actor.unwrap(); assert_eq!(actor.docs, vec!["Doc comment for service"]); - let IDLType::ServT(methods) = &actor.typ else { + let IDLType::ServT(methods) = &actor.typ.kind else { panic!("actor is not a service"); }; assert_eq!(methods[0].docs, vec!["Doc comment for f"]); @@ -82,9 +82,9 @@ service server : { } }) .unwrap(); - match &list.typ { + match &list.typ.kind { IDLType::OptT(list_inner) => { - let IDLType::RecordT(fields) = list_inner.as_ref() else { + let IDLType::RecordT(fields) = &list_inner.kind else { panic!("inner is not a record"); }; assert_eq!(fields[0].docs, vec!["Doc comment for List.head"]); diff --git a/rust/candid_parser/tests/test_doc_comments.rs b/rust/candid_parser/tests/test_doc_comments.rs index 53762f80b..ee714eaf0 100644 --- a/rust/candid_parser/tests/test_doc_comments.rs +++ b/rust/candid_parser/tests/test_doc_comments.rs @@ -41,7 +41,7 @@ type network = variant { assert_eq!(binding.id, "network"); // No field should have the inline comment - let fields = extract_variant_fields(&binding.typ); + let fields = extract_variant_fields(&binding.typ.kind); assert_eq!(fields.len(), 3); for field in fields { assert!( @@ -78,7 +78,7 @@ type network = variant { ); assert_eq!(binding.docs[0], "Doc comment for network"); - let fields = extract_variant_fields(&binding.typ); + let fields = extract_variant_fields(&binding.typ.kind); assert_eq!(fields.len(), 3); // Check that only the testnet field has the doc comment @@ -143,7 +143,7 @@ type my_type = variant { assert_eq!(binding.id, "my_type"); assert_eq!(binding.docs, vec!["Doc comment for my_type"]); - let fields = extract_variant_fields(&binding.typ); + let fields = extract_variant_fields(&binding.typ.kind); assert_eq!(fields.len(), 3); for field in fields { assert!( diff --git a/tools/didc/src/main.rs b/tools/didc/src/main.rs index f22ce6ca8..d77aaa441 100644 --- a/tools/didc/src/main.rs +++ b/tools/didc/src/main.rs @@ -131,7 +131,7 @@ impl TypeAnnotation { (Some(tys), None) => { let mut types = Vec::new(); for ty in tys.args.iter() { - types.push(ast_to_type(&env, ty)?); + types.push(ast_to_type(&env, &ty.kind)?); } Ok((env, types)) }