Merge pull request #43 from magicant/lexer_core

Handle line continuation in Lexer::peek_char

yash-syntax/src/parser/core.rs

@@ -584,16 +584,12 @@ impl Parser<'_> {
     /// [`take_token_auto`](Self::take_token_auto)) and before the next token is
     /// [peeked](Self::peek_token). Otherwise, this function would panic.
     ///
-    /// This function consumes and ignores line continuations that may lie
-    /// between the tokens.
-    ///
     /// # Panics
     ///
     /// If the previous token has not been taken or the next token has been
     /// peeked.
     pub async fn has_blank(&mut self) -> Result<bool> {
         assert!(self.token.is_none(), "There should be no pending token");
-        self.lexer.line_continuations().await?;
         let c = self.lexer.peek_char().await?;
         Ok(c.map_or(false, |c| is_blank(c.value)))
     }

yash-syntax/src/parser/lex/arith.rs

@@ -39,9 +39,6 @@ impl Lexer {
     /// The `location` parameter should be the location of the initial `$`. It
     /// is used to construct the result, but this function does not check if it
     /// actually is a location of `$`.
-    ///
-    /// This function does not consume line continuations between `$` and `(`.
-    /// Line continuations should have been consumed beforehand.
     pub async fn arithmetic_expansion(
         &mut self,
         location: Location,
@@ -52,7 +49,6 @@ impl Lexer {
         if !self.skip_if(|c| c == '(').await? {
             return Ok(Err(location));
         }
-        self.line_continuations().await?;
         if !self.skip_if(|c| c == '(').await? {
             self.rewind(index);
             return Ok(Err(location));
@@ -77,7 +73,6 @@ impl Lexer {
                 return Err(Error { cause, location });
             }
         }
-        self.line_continuations().await?;
         match self.peek_char().await? {
             Some(sc) if sc.value == ')' => self.consume_char(),
             Some(_) => {

yash-syntax/src/parser/lex/backquote.rs

@@ -24,13 +24,11 @@ use crate::syntax::BackquoteUnit;
 use crate::syntax::TextUnit;
 
 impl Lexer {
-    /// Parses a backquote unit, possibly preceded by line continuations.
+    /// Parses a backquote unit.
     async fn backquote_unit(
         &mut self,
         double_quote_escapable: bool,
     ) -> Result<Option<BackquoteUnit>> {
-        self.line_continuations().await?;
-
         if self.skip_if(|c| c == '\\').await? {
             let is_escapable =
                 |c| matches!(c, '$' | '`' | '\\') || c == '"' && double_quote_escapable;
@@ -55,10 +53,9 @@ impl Lexer {
     /// no closing backquote.
     ///
     /// Between the backquotes, only backslashes can have special meanings. A
-    /// backslash followed by a newline is parsed as line continuation. A
     /// backslash is an escape character if it precedes a dollar, backquote, or
-    /// another backslash. If `double_quote_escapable` is true, double quotes can
-    /// also be backslash-escaped.
+    /// another backslash. If `double_quote_escapable` is true, double quotes
+    /// can also be backslash-escaped.
     pub async fn backquote(&mut self, double_quote_escapable: bool) -> Result<Option<TextUnit>> {
         let location = match self.consume_char_if(|c| c == '`').await? {
             None => return Ok(None),

yash-syntax/src/parser/lex/braced_param.rs

@@ -39,13 +39,11 @@ pub fn is_name_char(c: char) -> bool {
 impl Lexer {
     /// Consumes a length prefix (`#`) if any.
     async fn length_prefix(&mut self) -> Result<bool> {
-        self.line_continuations().await?;
         let index = self.index();
         if !self.skip_if(|c| c == '#').await? {
             return Ok(false);
         }
 
-        self.line_continuations().await?;
         if let Some(&SourceChar { value: '}', .. }) = self.peek_char().await? {
             self.rewind(index);
             Ok(false)
@@ -54,13 +52,6 @@ impl Lexer {
         }
     }
 
-    /// Consumes a POSIXly-portable name character optionally preceded by line
-    /// continuations.
-    async fn consume_name_char(&mut self) -> Result<Option<&SourceChar>> {
-        self.line_continuations().await?;
-        self.consume_char_if(is_name_char).await
-    }
-
     /// Parses a parameter expansion that is enclosed in braces.
     ///
     /// The initial `$` must have been consumed before calling this function.
@@ -72,9 +63,6 @@ impl Lexer {
     /// The `location` parameter should be the location of the initial `$`. It
     /// is used to construct the result, but this function does not check if it
     /// actually is a location of `$`.
-    ///
-    /// This function does not consume line continuations after the initial `$`.
-    /// They should have been consumed beforehand.
     pub async fn braced_param(
         &mut self,
         location: Location,
@@ -85,8 +73,6 @@ impl Lexer {
 
         let has_length_prefix = self.length_prefix().await?;
 
-        self.line_continuations().await?;
-
         let sc = self.peek_char().await?.unwrap();
         let c = sc.value;
         let name = if is_special_parameter_char(c) {
@@ -95,7 +81,7 @@ impl Lexer {
         } else if is_name_char(c) {
             self.consume_char();
             let mut name = c.to_string();
-            while let Some(c) = self.consume_name_char().await? {
+            while let Some(c) = self.consume_char_if(is_name_char).await? {
                 name.push(c.value);
             }
             name
@@ -110,8 +96,6 @@ impl Lexer {
             return Err(Error { cause, location });
         };
 
-        self.line_continuations().await?;
-
         if !self.skip_if(|c| c == '}').await? {
             let opening_location = location;
             let cause = SyntaxError::UnclosedParam { opening_location }.into();

yash-syntax/src/parser/lex/command_subst.rs

@@ -36,9 +36,6 @@ impl Lexer {
     /// `opening_location` should be the location of the initial `$`. It is used
     /// to construct the result, but this function does not check if it actually
     /// is a location of `$`.
-    ///
-    /// This function does not consume line continuations between `$` and `(`.
-    /// Line continuations should have been consumed beforehand.
     pub async fn command_substitution(
         &mut self,
         opening_location: Location,