diff --git a/src/parser/hir/syntax_shape.rs b/src/parser/hir/syntax_shape.rs index 4ac6675f0d26..1a140d86bdc7 100644 --- a/src/parser/hir/syntax_shape.rs +++ b/src/parser/hir/syntax_shape.rs @@ -15,7 +15,7 @@ use crate::parser::PipelineElement; use crate::parser::{ hir, hir::{debug_tokens, TokensIterator}, - FlatShape, Operator, Pipeline, RawToken, TokenNode, + Operator, Pipeline, RawToken, TokenNode, }; use crate::prelude::*; use derive_new::new; @@ -39,6 +39,7 @@ pub(crate) use self::expression::variable_path::{ ExpressionContinuationShape, MemberShape, PathTailShape, VariablePathShape, }; pub(crate) use self::expression::{continue_expression, AnyExpressionShape}; +pub(crate) use self::flat_shape::FlatShape; #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SyntaxShape { @@ -66,30 +67,39 @@ impl FallibleColorSyntax for SyntaxShape { shapes: &mut Vec>, ) -> Result<(), ShellError> { match self { - SyntaxShape::Any => color_syntax(&AnyExpressionShape, token_nodes, context, shapes).1, + SyntaxShape::Any => { + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) + } SyntaxShape::List => { color_syntax(&ExpressionListShape, token_nodes, context, shapes); Ok(()) } - SyntaxShape::Int => color_syntax(&IntShape, token_nodes, context, shapes).1, - SyntaxShape::String => { - color_syntax_with( - &StringShape, - &FlatShape::String, - token_nodes, - context, - shapes, - ) - .1 + SyntaxShape::Int => color_fallible_syntax(&IntShape, token_nodes, context, shapes), + SyntaxShape::String => color_fallible_syntax_with( + &StringShape, + &FlatShape::String, + token_nodes, + context, + shapes, + ), + SyntaxShape::Member => { + color_fallible_syntax(&MemberShape, token_nodes, context, shapes) } - SyntaxShape::Member => color_syntax(&MemberShape, token_nodes, context, shapes).1, SyntaxShape::ColumnPath => { - color_syntax(&ColumnPathShape, token_nodes, context, shapes).1 + color_fallible_syntax(&ColumnPathShape, token_nodes, context, shapes) + } + SyntaxShape::Number => { + color_fallible_syntax(&NumberShape, token_nodes, context, shapes) + } + SyntaxShape::Path => { + color_fallible_syntax(&FilePathShape, token_nodes, context, shapes) + } + SyntaxShape::Pattern => { + color_fallible_syntax(&PatternShape, token_nodes, context, shapes) + } + SyntaxShape::Block => { + color_fallible_syntax(&AnyBlockShape, token_nodes, context, shapes) } - SyntaxShape::Number => color_syntax(&NumberShape, token_nodes, context, shapes).1, - SyntaxShape::Path => color_syntax(&FilePathShape, token_nodes, context, shapes).1, - SyntaxShape::Pattern => color_syntax(&PatternShape, token_nodes, context, shapes).1, - SyntaxShape::Block => color_syntax(&AnyBlockShape, token_nodes, context, shapes).1, } } } @@ -218,23 +228,23 @@ pub trait ColorSyntax: std::fmt::Debug + Copy { ) -> Self::Info; } -impl ColorSyntax for T -where - T: FallibleColorSyntax, -{ - type Info = Result; - type Input = T::Input; - - fn color_syntax<'a, 'b>( - &self, - input: &Self::Input, - token_nodes: &'b mut TokensIterator<'a>, - context: &ExpandContext, - shapes: &mut Vec>, - ) -> Result { - FallibleColorSyntax::color_syntax(self, input, token_nodes, context, shapes) - } -} +// impl ColorSyntax for T +// where +// T: FallibleColorSyntax, +// { +// type Info = Result; +// type Input = T::Input; + +// fn color_syntax<'a, 'b>( +// &self, +// input: &Self::Input, +// token_nodes: &'b mut TokensIterator<'a>, +// context: &ExpandContext, +// shapes: &mut Vec>, +// ) -> Result { +// FallibleColorSyntax::color_syntax(self, input, token_nodes, context, shapes) +// } +// } pub(crate) trait ExpandSyntax: std::fmt::Debug + Copy { type Output: std::fmt::Debug; @@ -358,6 +368,40 @@ pub fn color_syntax_with<'a, 'b, T: ColorSyntax, U, I>( ((), result) } +pub fn color_fallible_syntax_with<'a, 'b, T: FallibleColorSyntax, U, I>( + shape: &T, + input: &I, + token_nodes: &'b mut TokensIterator<'a>, + context: &ExpandContext, + shapes: &mut Vec>, +) -> Result { + trace!(target: "nu::color_syntax", "before {} :: {:?}", std::any::type_name::(), debug_tokens(token_nodes, context.source)); + + if token_nodes.at_end() { + trace!(target: "nu::color_syntax", "at eof"); + return Err(ShellError::unexpected_eof("coloring", Tag::unknown())); + } + + let len = shapes.len(); + let result = shape.color_syntax(input, token_nodes, context, shapes); + + trace!(target: "nu::color_syntax", "ok :: {:?}", debug_tokens(token_nodes, context.source)); + + if log_enabled!(target: "nu::color_syntax", log::Level::Trace) { + trace!(target: "nu::color_syntax", "after {}", std::any::type_name::()); + + if len < shapes.len() { + for i in len..(shapes.len()) { + trace!(target: "nu::color_syntax", "new shape :: {:?}", shapes[i]); + } + } else { + trace!(target: "nu::color_syntax", "no new shapes"); + } + } + + result +} + pub(crate) fn expand_expr<'a, 'b, T: ExpandExpression>( shape: &T, token_nodes: &'b mut TokensIterator<'a>, @@ -670,7 +714,7 @@ impl FallibleColorSyntax for CommandHeadShape { shapes: &mut Vec>, ) -> Result { // If we don't ultimately find a token, roll back - token_nodes.checkpoint_with(|token_nodes| { + token_nodes.atomic(|token_nodes| { // First, take a look at the next token let atom = expand_atom( token_nodes, @@ -1197,7 +1241,7 @@ impl ColorSyntax for CommandShape { context: &ExpandContext, shapes: &mut Vec>, ) { - let (_, kind) = color_syntax(&CommandHeadShape, token_nodes, context, shapes); + let kind = color_fallible_syntax(&CommandHeadShape, token_nodes, context, shapes); match kind { Err(_) => { diff --git a/src/parser/hir/syntax_shape/block.rs b/src/parser/hir/syntax_shape/block.rs index 18e27e5ea3bd..806681691eca 100644 --- a/src/parser/hir/syntax_shape/block.rs +++ b/src/parser/hir/syntax_shape/block.rs @@ -2,10 +2,10 @@ use crate::errors::ShellError; use crate::parser::{ hir, hir::syntax_shape::{ - color_fallible_syntax, color_syntax, color_syntax_with, continue_expression, expand_expr, - expand_syntax, DelimitedShape, ExpandContext, ExpandExpression, - ExpressionContinuationShape, ExpressionListShape, FallibleColorSyntax, FlatShape, - MemberShape, PathTailShape, VariablePathShape, + color_fallible_syntax, color_syntax_with, continue_expression, expand_expr, expand_syntax, + DelimitedShape, ExpandContext, ExpandExpression, ExpressionContinuationShape, + ExpressionListShape, FallibleColorSyntax, FlatShape, MemberShape, PathTailShape, + VariablePathShape, }, hir::tokens_iterator::TokensIterator, parse::token_tree::Delimiter, @@ -55,7 +55,7 @@ impl FallibleColorSyntax for AnyBlockShape { } // Otherwise, look for a shorthand block. If none found, fail - color_syntax(&ShorthandBlock, token_nodes, context, shapes).1 + color_fallible_syntax(&ShorthandBlock, token_nodes, context, shapes) } } @@ -150,8 +150,8 @@ impl FallibleColorSyntax for ShorthandPath { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - token_nodes.checkpoint_with(|token_nodes| { - let variable = color_syntax(&VariablePathShape, token_nodes, context, shapes).1; + token_nodes.atomic(|token_nodes| { + let variable = color_fallible_syntax(&VariablePathShape, token_nodes, context, shapes); match variable { Ok(_) => { @@ -165,11 +165,11 @@ impl FallibleColorSyntax for ShorthandPath { } // look for a member (`` -> `$it.`) - color_syntax(&MemberShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; // Now that we've synthesized the head, of the path, proceed to expand the tail of the path // like any other path. - let tail = color_syntax(&PathTailShape, token_nodes, context, shapes).1; + let tail = color_fallible_syntax(&PathTailShape, token_nodes, context, shapes); match tail { Ok(_) => {} diff --git a/src/parser/hir/syntax_shape/expression.rs b/src/parser/hir/syntax_shape/expression.rs index 9ff8d08f5fba..fc99c38dc370 100644 --- a/src/parser/hir/syntax_shape/expression.rs +++ b/src/parser/hir/syntax_shape/expression.rs @@ -9,9 +9,9 @@ pub(crate) mod unit; pub(crate) mod variable_path; use crate::parser::hir::syntax_shape::{ - color_delimited_square, color_syntax, color_syntax_with, expand_atom, expand_delimited_square, - expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape, DotShape, ExpandContext, - ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation, + color_delimited_square, color_fallible_syntax, color_fallible_syntax_with, expand_atom, + expand_delimited_square, expand_expr, expand_syntax, AtomicToken, BareShape, ColorableDotShape, + DotShape, ExpandContext, ExpandExpression, ExpandSyntax, ExpansionRule, ExpressionContinuation, ExpressionContinuationShape, FallibleColorSyntax, FlatShape, }; use crate::parser::{ @@ -49,7 +49,7 @@ impl FallibleColorSyntax for AnyExpressionShape { shapes: &mut Vec>, ) -> Result<(), ShellError> { // Look for an expression at the cursor - color_syntax(&AnyExpressionStartShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&AnyExpressionStartShape, token_nodes, context, shapes)?; match continue_coloring_expression(token_nodes, context, shapes) { Err(_) => { @@ -97,11 +97,12 @@ pub(crate) fn continue_coloring_expression( shapes: &mut Vec>, ) -> Result<(), ShellError> { // if there's not even one expression continuation, fail - color_syntax(&ExpressionContinuationShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context, shapes)?; loop { // Check to see whether there's any continuation after the head expression - let result = color_syntax(&ExpressionContinuationShape, token_nodes, context, shapes).1; + let result = + color_fallible_syntax(&ExpressionContinuationShape, token_nodes, context, shapes); match result { Err(_) => { @@ -223,8 +224,13 @@ impl FallibleColorSyntax for BareTailShape { let len = shapes.len(); loop { - let word = - color_syntax_with(&BareShape, &FlatShape::Word, token_nodes, context, shapes).1; + let word = color_fallible_syntax_with( + &BareShape, + &FlatShape::Word, + token_nodes, + context, + shapes, + ); match word { // if a word was found, continue @@ -234,14 +240,13 @@ impl FallibleColorSyntax for BareTailShape { } // try to find a dot - let dot = color_syntax_with( + let dot = color_fallible_syntax_with( &ColorableDotShape, &FlatShape::Word, token_nodes, context, shapes, - ) - .1; + ); match dot { // if a dot was found, try to find another word diff --git a/src/parser/hir/syntax_shape/expression/atom.rs b/src/parser/hir/syntax_shape/expression/atom.rs index 7f0e3e094e4c..83306da74182 100644 --- a/src/parser/hir/syntax_shape/expression/atom.rs +++ b/src/parser/hir/syntax_shape/expression/atom.rs @@ -180,30 +180,30 @@ impl<'tokens> TaggedAtomicToken<'tokens> { pub(crate) fn color_tokens(&self, shapes: &mut Vec>) { match &self.item { AtomicToken::Eof { .. } => {} - AtomicToken::Error { error } => return shapes.push(FlatShape::Error.tagged(error.tag)), - AtomicToken::Operator { text } => { - return shapes.push(FlatShape::Operator.tagged(text)); + AtomicToken::Error { .. } => return shapes.push(FlatShape::Error.tagged(self.tag)), + AtomicToken::Operator { .. } => { + return shapes.push(FlatShape::Operator.tagged(self.tag)); } - AtomicToken::ShorthandFlag { name } => { - return shapes.push(FlatShape::ShorthandFlag.tagged(name)); + AtomicToken::ShorthandFlag { .. } => { + return shapes.push(FlatShape::ShorthandFlag.tagged(self.tag)); } - AtomicToken::LonghandFlag { name } => { - return shapes.push(FlatShape::Flag.tagged(name)); + AtomicToken::LonghandFlag { .. } => { + return shapes.push(FlatShape::Flag.tagged(self.tag)); } - AtomicToken::Whitespace { text } => { - return shapes.push(FlatShape::Whitespace.tagged(text)); + AtomicToken::Whitespace { .. } => { + return shapes.push(FlatShape::Whitespace.tagged(self.tag)); } - AtomicToken::FilePath { path } => return shapes.push(FlatShape::Path.tagged(path)), - AtomicToken::Dot { text } => return shapes.push(FlatShape::Dot.tagged(text)), + AtomicToken::FilePath { .. } => return shapes.push(FlatShape::Path.tagged(self.tag)), + AtomicToken::Dot { .. } => return shapes.push(FlatShape::Dot.tagged(self.tag)), AtomicToken::Number { - number: RawNumber::Decimal(tag), + number: RawNumber::Decimal(_), } => { - return shapes.push(FlatShape::Decimal.tagged(tag)); + return shapes.push(FlatShape::Decimal.tagged(self.tag)); } AtomicToken::Number { - number: RawNumber::Int(tag), + number: RawNumber::Int(_), } => { - return shapes.push(FlatShape::Int.tagged(tag)); + return shapes.push(FlatShape::Int.tagged(self.tag)); } AtomicToken::Size { number, unit } => { return shapes.push( @@ -214,33 +214,24 @@ impl<'tokens> TaggedAtomicToken<'tokens> { .tagged(self.tag), ); } - AtomicToken::String { body } => return shapes.push(FlatShape::String.tagged(body)), - AtomicToken::ItVariable { name } => { - return shapes.push(FlatShape::ItVariable.tagged(name)) - } - AtomicToken::Variable { name } => return shapes.push(FlatShape::Variable.tagged(name)), - AtomicToken::ExternalCommand { command } => { - return shapes.push(FlatShape::ExternalCommand.tagged(command)); - } - AtomicToken::ExternalWord { text } => { - return shapes.push(FlatShape::ExternalWord.tagged(text)) + AtomicToken::String { .. } => return shapes.push(FlatShape::String.tagged(self.tag)), + AtomicToken::ItVariable { .. } => { + return shapes.push(FlatShape::ItVariable.tagged(self.tag)) } - AtomicToken::GlobPattern { pattern } => { - return shapes.push(FlatShape::GlobPattern.tagged(pattern)) + AtomicToken::Variable { .. } => { + return shapes.push(FlatShape::Variable.tagged(self.tag)) } - AtomicToken::Word { text } => return shapes.push(FlatShape::Word.tagged(text)), - AtomicToken::SquareDelimited { .. } => { - unreachable!("BUG: handle nested tokens before calling color_tokens") + AtomicToken::ExternalCommand { .. } => { + return shapes.push(FlatShape::ExternalCommand.tagged(self.tag)); } - AtomicToken::ParenDelimited { .. } => { - unreachable!("BUG: handle nested tokens before calling color_tokens") + AtomicToken::ExternalWord { .. } => { + return shapes.push(FlatShape::ExternalWord.tagged(self.tag)) } - AtomicToken::BraceDelimited { .. } => { - unreachable!("BUG: handle nested tokens before calling color_tokens") - } - AtomicToken::Pipeline { .. } => { - unreachable!("BUG: handle nested tokens before calling color_tokens") + AtomicToken::GlobPattern { .. } => { + return shapes.push(FlatShape::GlobPattern.tagged(self.tag)) } + AtomicToken::Word { .. } => return shapes.push(FlatShape::Word.tagged(self.tag)), + _ => return shapes.push(FlatShape::Error.tagged(self.tag)), } } } @@ -431,37 +422,14 @@ pub fn expand_atom<'me, 'content>( // handle this next } - TokenNode::Call(_) => unimplemented!("expand_atom TokenNode::Call"), - TokenNode::Nodes(_) => unimplemented!("expand_atom TokenNode::Nodes"), - TokenNode::Pipeline(_) => unimplemented!("expand_atom TokenNode::Pipeline"), - TokenNode::Error(error) => { + peeked.commit(); return Ok(AtomicToken::Error { error: error.clone(), } - .tagged(error.tag)) + .tagged(error.tag)); } - // { ... } - TokenNode::Delimited(Tagged { - item: - DelimitedNode { - delimiter: Delimiter::Brace, - .. - }, - .. - }) => unimplemented!("expand_atom TokenNode::Delimited (brace)"), - - // ( ... ) - TokenNode::Delimited(Tagged { - item: - DelimitedNode { - delimiter: Delimiter::Paren, - .. - }, - .. - }) => unimplemented!("expand_atom TokenNode::Delimited (paren)"), - // [ ... ] TokenNode::Delimited(Tagged { item: @@ -520,6 +488,16 @@ pub fn expand_atom<'me, 'content>( )) } }, + + other => { + let tag = peeked.node.tag(); + + peeked.commit(); + return Ok(AtomicToken::Error { + error: ShellError::type_error("token", other.tagged_type_name()).tagged(tag), + } + .tagged(tag)); + } } parse_single_node(token_nodes, expected, |token, token_tag, err| { diff --git a/src/parser/hir/syntax_shape/expression/list.rs b/src/parser/hir/syntax_shape/expression/list.rs index 47ed909088c0..4109108a3795 100644 --- a/src/parser/hir/syntax_shape/expression/list.rs +++ b/src/parser/hir/syntax_shape/expression/list.rs @@ -2,8 +2,9 @@ use crate::errors::ShellError; use crate::parser::{ hir, hir::syntax_shape::{ - color_syntax, expand_atom, expand_expr, maybe_spaced, spaced, AnyExpressionShape, - ColorSyntax, ExpandContext, ExpandSyntax, ExpansionRule, SpaceShape, + color_fallible_syntax, color_syntax, expand_atom, expand_expr, maybe_spaced, spaced, + AnyExpressionShape, ColorSyntax, ExpandContext, ExpandSyntax, ExpansionRule, + MaybeSpaceShape, SpaceShape, }, hir::TokensIterator, FlatShape, @@ -65,6 +66,9 @@ impl ColorSyntax for ExpressionListShape { // coloring mode") let mut backoff = false; + // Consume any leading whitespace + color_syntax(&MaybeSpaceShape, token_nodes, context, shapes); + loop { // If we reached the very end of the token stream, we're done if token_nodes.at_end() { @@ -74,7 +78,7 @@ impl ColorSyntax for ExpressionListShape { if backoff { let len = shapes.len(); - // If we previously encountered a parsing error, switch to backoff coloring mode + // If we previously encountered a parsing error, use backoff coloring mode color_syntax(&SimplestExpression, token_nodes, context, shapes); if len == shapes.len() && !token_nodes.at_end() { @@ -82,14 +86,26 @@ impl ColorSyntax for ExpressionListShape { panic!("Unexpected tokens left that couldn't be colored even with SimplestExpression") } } else { - match color_syntax(&SpaceShape, token_nodes, context, shapes).1 { - Err(_) => backoff = true, + // Try to color the head of the stream as an expression + match color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes) { + // If no expression was found, switch to backoff coloring mode + Err(_) => { + backoff = true; + continue; + } + Ok(_) => {} + } + + // If an expression was found, consume a space + match color_fallible_syntax(&SpaceShape, token_nodes, context, shapes) { + Err(_) => { + // If no space was found, we're either at the end or there's an error. + // Either way, switch to backoff coloring mode. If we're at the end + // it won't have any consequences. + backoff = true; + } Ok(_) => { - // Otherwise, try to color the head of the stream as an expression - match color_syntax(&AnyExpressionShape, token_nodes, context, shapes).1 { - Err(_) => backoff = true, - Ok(_) => {} - } + // Otherwise, move on to the next expression } } } diff --git a/src/parser/hir/syntax_shape/expression/pattern.rs b/src/parser/hir/syntax_shape/expression/pattern.rs index 4a2dae90934c..5c863de72859 100644 --- a/src/parser/hir/syntax_shape/expression/pattern.rs +++ b/src/parser/hir/syntax_shape/expression/pattern.rs @@ -20,7 +20,7 @@ impl FallibleColorSyntax for PatternShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - token_nodes.checkpoint_with(|token_nodes| { + token_nodes.atomic(|token_nodes| { let atom = expand_atom(token_nodes, "pattern", context, ExpansionRule::permissive())?; match &atom.item { diff --git a/src/parser/hir/syntax_shape/expression/variable_path.rs b/src/parser/hir/syntax_shape/expression/variable_path.rs index 1594ccf9e6d6..a9de92de4b91 100644 --- a/src/parser/hir/syntax_shape/expression/variable_path.rs +++ b/src/parser/hir/syntax_shape/expression/variable_path.rs @@ -1,8 +1,8 @@ use crate::parser::hir::syntax_shape::{ - color_syntax, color_syntax_with, expand_atom, expand_expr, expand_syntax, parse_single_node, - AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression, ExpandSyntax, - ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape, TestSyntax, - WhitespaceShape, + color_fallible_syntax, color_fallible_syntax_with, expand_atom, expand_expr, expand_syntax, + parse_single_node, AnyExpressionShape, AtomicToken, BareShape, ExpandContext, ExpandExpression, + ExpandSyntax, ExpansionRule, FallibleColorSyntax, FlatShape, Peeked, SkipSyntax, StringShape, + TestSyntax, WhitespaceShape, }; use crate::parser::{hir, hir::Expression, hir::TokensIterator, Operator, RawToken}; use crate::prelude::*; @@ -55,20 +55,19 @@ impl FallibleColorSyntax for VariablePathShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - token_nodes.checkpoint_with(|token_nodes| { + token_nodes.atomic(|token_nodes| { // If the head of the token stream is not a variable, fail - color_syntax(&VariableShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&VariableShape, token_nodes, context, shapes)?; loop { // look for a dot at the head of a stream - let dot = color_syntax_with( + let dot = color_fallible_syntax_with( &ColorableDotShape, &FlatShape::Dot, token_nodes, context, shapes, - ) - .1; + ); // if there's no dot, we're done match dot { @@ -77,7 +76,7 @@ impl FallibleColorSyntax for VariablePathShape { } // otherwise, look for a member, and if you don't find one, fail - color_syntax(&MemberShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; } Ok(()) @@ -100,15 +99,14 @@ impl FallibleColorSyntax for PathTailShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - token_nodes.checkpoint_with(|token_nodes| loop { - let result = color_syntax_with( + token_nodes.atomic(|token_nodes| loop { + let result = color_fallible_syntax_with( &ColorableDotShape, &FlatShape::Dot, token_nodes, context, shapes, - ) - .1; + ); match result { Err(_) => return Ok(()), @@ -116,7 +114,7 @@ impl FallibleColorSyntax for PathTailShape { } // If we've seen a dot but not a member, fail - color_syntax(&MemberShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; }) } } @@ -213,32 +211,33 @@ impl FallibleColorSyntax for ExpressionContinuationShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result { - token_nodes.checkpoint_with(|token_nodes| { + token_nodes.atomic(|token_nodes| { // Try to expand a `.` - let dot = color_syntax_with( + let dot = color_fallible_syntax_with( &ColorableDotShape, &FlatShape::Dot, token_nodes, context, shapes, - ) - .1; + ); match dot { Ok(_) => { // we found a dot, so let's keep looking for a member; if no member was found, fail - color_syntax(&MemberShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; Ok(ContinuationInfo::Dot) } Err(_) => { - // we didn't find a dot, so let's see if we're looking at an infix. If not found, fail - color_syntax(&InfixShape, token_nodes, context, shapes).1?; + token_nodes.atomic(|token_nodes| { + // we didn't find a dot, so let's see if we're looking at an infix. If not found, fail + color_fallible_syntax(&InfixShape, token_nodes, context, shapes)?; - // now that we've seen an infix shape, look for any expression. If not found, fail - color_syntax(&AnyExpressionShape, token_nodes, context, shapes).1?; + // now that we've seen an infix shape, look for any expression. If not found, fail + color_fallible_syntax(&AnyExpressionShape, token_nodes, context, shapes)?; - Ok(ContinuationInfo::Infix) + Ok(ContinuationInfo::Infix) + }) } } }) @@ -445,27 +444,26 @@ impl FallibleColorSyntax for ColumnPathShape { shapes: &mut Vec>, ) -> Result<(), ShellError> { // If there's not even one member shape, fail - color_syntax(&MemberShape, token_nodes, context, shapes).1?; + color_fallible_syntax(&MemberShape, token_nodes, context, shapes)?; loop { let checkpoint = token_nodes.checkpoint(); - match color_syntax_with( + match color_fallible_syntax_with( &ColorableDotShape, &FlatShape::Dot, checkpoint.iterator, context, shapes, - ) - .1 - { + ) { Err(_) => { // we already saw at least one member shape, so return successfully return Ok(()); } Ok(_) => { - match color_syntax(&MemberShape, checkpoint.iterator, context, shapes).1 { + match color_fallible_syntax(&MemberShape, checkpoint.iterator, context, shapes) + { Err(_) => { // we saw a dot but not a member (but we saw at least one member), // so don't commit the dot but return successfully @@ -509,7 +507,7 @@ impl FallibleColorSyntax for MemberShape { context: &ExpandContext, shapes: &mut Vec>, ) -> Result<(), ShellError> { - let (_, bare) = color_syntax_with( + let bare = color_fallible_syntax_with( &BareShape, &FlatShape::BareMember, token_nodes, @@ -525,14 +523,13 @@ impl FallibleColorSyntax for MemberShape { } // Look for a string token. If we don't find one, fail - color_syntax_with( + color_fallible_syntax_with( &StringShape, &FlatShape::StringMember, token_nodes, context, shapes, ) - .1 } } @@ -586,12 +583,11 @@ impl FallibleColorSyntax for ColorableDotShape { node if node.is_dot() => { peeked.commit(); shapes.push((*input).tagged(node.tag())); + Ok(()) } - _ => return Ok(()), + other => Err(ShellError::type_error("dot", other.tagged_type_name())), } - - Ok(()) } } @@ -647,32 +643,31 @@ impl FallibleColorSyntax for InfixShape { let mut shapes = vec![]; // An infix operator must be prefixed by whitespace. If no whitespace was found, fail - color_syntax(&WhitespaceShape, checkpoint.iterator, context, &mut shapes).1?; + color_fallible_syntax(&WhitespaceShape, checkpoint.iterator, context, &mut shapes)?; // Parse the next TokenNode after the whitespace - let operator = parse_single_node( + parse_single_node( checkpoint.iterator, "infix operator", |token, token_tag, _| { - Ok(match token { + match token { // If it's an operator (and not `.`), it's a match RawToken::Operator(operator) if operator != Operator::Dot => { - shapes.push(FlatShape::Operator.tagged(token_tag)) + shapes.push(FlatShape::Operator.tagged(token_tag)); + Ok(()) } // Otherwise, it's not a match - _ => {} - }) + _ => Err(ShellError::type_error( + "infix operator", + token.type_name().tagged(token_tag), + )), + } }, - ); - - match operator { - Err(_) => return Ok(()), - Ok(_) => {} - } + )?; // An infix operator must be followed by whitespace. If no whitespace was found, fail - color_syntax(&WhitespaceShape, checkpoint.iterator, context, &mut shapes).1?; + color_fallible_syntax(&WhitespaceShape, checkpoint.iterator, context, &mut shapes)?; outer_shapes.extend(shapes); checkpoint.commit(); diff --git a/src/parser/hir/syntax_shape/flat_shape.rs b/src/parser/hir/syntax_shape/flat_shape.rs index 19b1546d1c86..dc712e8841e2 100644 --- a/src/parser/hir/syntax_shape/flat_shape.rs +++ b/src/parser/hir/syntax_shape/flat_shape.rs @@ -67,7 +67,7 @@ impl FlatShape { } TokenNode::Pipeline(pipeline) => { for part in &pipeline.parts { - if let Some(_pipe) = part.pipe { + if let Some(_) = part.pipe { shapes.push(FlatShape::Pipe.tagged(part.tag)); } } @@ -88,7 +88,7 @@ impl FlatShape { }, tag, }) => shapes.push(FlatShape::ShorthandFlag.tagged(tag)), - TokenNode::Whitespace(_) => unimplemented!(), + TokenNode::Whitespace(v) => unimplemented!(), TokenNode::Error(v) => shapes.push(FlatShape::Error.tagged(v.tag)), } } diff --git a/src/parser/hir/tokens_iterator.rs b/src/parser/hir/tokens_iterator.rs index 96517bdb67ab..f597c850bd28 100644 --- a/src/parser/hir/tokens_iterator.rs +++ b/src/parser/hir/tokens_iterator.rs @@ -172,7 +172,7 @@ impl<'content> TokensIterator<'content> { /// Use a checkpoint when you need to peek more than one token ahead, but can't be sure /// that you'll succeed. - pub fn checkpoint_with<'me, T>( + pub fn atomic<'me, T>( &'me mut self, block: impl FnOnce(&mut TokensIterator<'content>) -> Result, ) -> Result { diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index 9ffbd26ecafd..73833f7be5f1 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -189,7 +189,7 @@ pub fn raw_number(input: NomSpan) -> IResult> { match input.fragment.chars().next() { None => return Ok((input, RawNumber::int((start, input.offset, input.extra)))), Some('.') => (), - Some(other) if other.is_whitespace() => { + other if is_boundary(other) => { return Ok((input, RawNumber::int((start, input.offset, input.extra)))) } _ => { @@ -215,16 +215,14 @@ pub fn raw_number(input: NomSpan) -> IResult> { let next = input.fragment.chars().next(); - if let Some(next) = next { - if !next.is_whitespace() { - return Err(nom::Err::Error(nom::error::make_error( - input, - nom::error::ErrorKind::Tag, - ))); - } + if is_boundary(next) { + Ok((input, RawNumber::decimal((start, end, input.extra)))) + } else { + Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::Tag, + ))) } - - Ok((input, RawNumber::decimal((start, end, input.extra)))) } #[tracable_parser] diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index 385c6b8cf5a2..efebc6737456 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -1,7 +1,7 @@ use crate::errors::{ArgumentError, ShellError}; use crate::parser::hir::syntax_shape::{ - color_syntax, expand_expr, flat_shape::FlatShape, spaced, BackoffColoringMode, ColorSyntax, - MaybeSpaceShape, + color_fallible_syntax, color_syntax, expand_expr, flat_shape::FlatShape, spaced, + BackoffColoringMode, ColorSyntax, MaybeSpaceShape, }; use crate::parser::registry::{NamedType, PositionalType, Signature}; use crate::parser::TokensIterator; @@ -201,7 +201,7 @@ impl ColorSyntax for CommandTailShape { trace_remaining("nodes", token_nodes.clone(), context.source()); for (name, kind) in &signature.named { - trace!(target: "nu::parse", "looking for {} : {:?}", name, kind); + trace!(target: "nu::color_syntax", "looking for {} : {:?}", name, kind); match kind { NamedType::Switch => { @@ -231,16 +231,19 @@ impl ColorSyntax for CommandTailShape { continue; } - color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); - match color_syntax(syntax_type, token_nodes, context, &mut shapes).1 { - Err(_) => { - // this is ok; it means the part after a mandatory flag isn't - // a valid expression, but that can happen while typing and - // we can live with it - } - - Ok(_) => {} - } + // We can live with unmatched syntax after a mandatory flag + token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If the part after a mandatory flag isn't present, that's ok, but we + // should roll back any whitespace we chomped + color_fallible_syntax( + syntax_type, + token_nodes, + context, + &mut shapes, + ) + }); args.insert(pos, shapes); token_nodes.restart(); @@ -262,16 +265,19 @@ impl ColorSyntax for CommandTailShape { continue; } - color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); - match color_syntax(syntax_type, token_nodes, context, &mut shapes).1 { - Err(_) => { - // this is ok; it means the part after an optional flag isn't - // a valid expression, but that can happen while typing and - // we can live with it - } - - Ok(_) => {} - } + // We can live with unmatched syntax after an optional flag + let _ = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If the part after a mandatory flag isn't present, that's ok, but we + // should roll back any whitespace we chomped + color_fallible_syntax( + syntax_type, + token_nodes, + context, + &mut shapes, + ) + }); args.insert(pos, shapes); token_nodes.restart(); @@ -293,7 +299,7 @@ impl ColorSyntax for CommandTailShape { match arg { PositionalType::Mandatory(..) => { if token_nodes.at_end() { - return; + break; } } @@ -310,16 +316,23 @@ impl ColorSyntax for CommandTailShape { match pos { None => break, Some(pos) => { - color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); - match color_syntax(&arg.syntax_type(), token_nodes, context, &mut shapes).1 { - Err(_) => { - // There are more tokens and we were expecting a positional argument here, - // but it didn't match. Hopefully it will be matched by a future token - } - Ok(_) => { - args.insert(pos, shapes); - } - } + // We can live with an unmatched positional argument. Hopefully it will be + // matched by a future token + let _ = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If no match, we should roll back any whitespace we chomped + color_fallible_syntax( + &arg.syntax_type(), + token_nodes, + context, + &mut shapes, + )?; + + args.insert(pos, shapes); + + Ok(()) + }); } } } @@ -338,18 +351,23 @@ impl ColorSyntax for CommandTailShape { None => break, Some(pos) => { let mut shapes = vec![]; - color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); - match color_syntax(&syntax_type, token_nodes, context, &mut shapes).1 { - Err(_) => { - // There are more tokens and we were expecting a positional argument here, - // but it didn't match. We're going to have to fall back to consuming the - // token with backoff coloring mode - break; - } - Ok(_) => {} + // If any arguments don't match, we'll fall back to backoff coloring mode + let result = token_nodes.atomic(|token_nodes| { + color_syntax(&MaybeSpaceShape, token_nodes, context, &mut shapes); + + // If no match, we should roll back any whitespace we chomped + color_fallible_syntax(&syntax_type, token_nodes, context, &mut shapes)?; + + args.insert(pos, shapes); + + Ok(()) + }); + + match result { + Err(_) => break, + Ok(_) => continue, } - args.insert(pos, shapes); } } } @@ -359,6 +377,8 @@ impl ColorSyntax for CommandTailShape { // Consume any remaining tokens with backoff coloring mode color_syntax(&BackoffColoringMode, token_nodes, context, shapes); + + shapes.sort_by(|a, b| a.tag.span.start().cmp(&b.tag.span.start())); } } diff --git a/src/shell/helper.rs b/src/shell/helper.rs index dd55cedc82b3..8707a1709e18 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -1,5 +1,5 @@ use crate::context::Context; -use crate::parser::hir::syntax_shape::{color_syntax, PipelineShape}; +use crate::parser::hir::syntax_shape::{color_fallible_syntax, FlatShape, PipelineShape}; use crate::parser::hir::TokensIterator; use crate::parser::nom_input; use crate::parser::parse::token_tree::TokenNode; @@ -7,6 +7,7 @@ use crate::parser::parse::tokens::RawToken; use crate::parser::{Pipeline, PipelineElement}; use crate::{Tag, Tagged, TaggedItem, Text}; use ansi_term::Color; +use log::trace; use rustyline::completion::Completer; use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; @@ -80,7 +81,7 @@ impl Highlighter for Helper { }; let Pipeline { parts } = pipeline.clone(); - let mut iter = parts.into_iter(); + let iter = parts.into_iter(); let tokens = vec![TokenNode::Pipeline(pipeline.clone().tagged(v.tag()))]; let mut tokens = TokensIterator::all(&tokens[..], v.tag()); @@ -92,26 +93,32 @@ impl Highlighter for Helper { let mut shapes = vec![]; // We just constructed a token list that only contains a pipeline, so it can't fail - color_syntax(&PipelineShape, &mut tokens, &expand_context, &mut shapes) - .1 + color_fallible_syntax(&PipelineShape, &mut tokens, &expand_context, &mut shapes) .unwrap(); - println!( - "\n{:?}", + trace!(target: "nu::shapes", + "SHAPES :: {:?}", shapes.iter().map(|shape| shape.item).collect::>() ); - loop { - match iter.next() { - None => { - return Cow::Owned(out); - } - Some(token) => { - let styled = paint_pipeline_element(&token, line); - out.push_str(&styled.to_string()); - } - } + for shape in shapes { + let styled = paint_flat_shape(shape, line); + out.push_str(&styled); } + + Cow::Owned(out) + + // loop { + // match iter.next() { + // None => { + // return Cow::Owned(out); + // } + // Some(token) => { + // let styled = paint_pipeline_element(&token, line); + // out.push_str(&styled.to_string()); + // } + // } + // } } } } @@ -133,6 +140,45 @@ fn vec_tag(input: Vec>) -> Option { }) } +fn paint_flat_shape(flat_shape: Tagged, line: &str) -> String { + let style = match &flat_shape.item { + FlatShape::OpenDelimiter(_) => Color::White.normal(), + FlatShape::CloseDelimiter(_) => Color::White.normal(), + FlatShape::ItVariable => Color::Purple.bold(), + FlatShape::Variable => Color::Purple.normal(), + FlatShape::Operator => Color::Yellow.normal(), + FlatShape::Dot => Color::White.normal(), + FlatShape::InternalCommand => Color::Cyan.bold(), + FlatShape::ExternalCommand => Color::Cyan.normal(), + FlatShape::ExternalWord => Color::Black.bold(), + FlatShape::BareMember => Color::Yellow.bold(), + FlatShape::StringMember => Color::Yellow.bold(), + FlatShape::String => Color::Green.normal(), + FlatShape::Path => Color::Cyan.normal(), + FlatShape::GlobPattern => Color::Cyan.bold(), + FlatShape::Word => Color::Green.normal(), + FlatShape::Pipe => Color::Purple.bold(), + FlatShape::Flag => Color::Black.bold(), + FlatShape::ShorthandFlag => Color::Black.bold(), + FlatShape::Int => Color::Purple.bold(), + FlatShape::Decimal => Color::Purple.bold(), + FlatShape::Whitespace => Color::White.normal(), + FlatShape::Error => Color::Red.bold(), + FlatShape::Size { number, unit } => { + let number = number.slice(line); + let unit = unit.slice(line); + return format!( + "{}{}", + Color::Purple.bold().paint(number), + Color::Yellow.paint(unit) + ); + } + }; + + let body = flat_shape.tag.slice(line); + style.paint(body).to_string() +} + fn paint_token_node(token_node: &TokenNode, line: &str) -> String { let styled = match token_node { TokenNode::Call(..) => Color::Cyan.bold().paint(token_node.tag().slice(line)),