[DNM] Parse /.../ regex literals (by re-lexing)

include/swift/AST/DiagnosticsParse.def

@@ -94,6 +94,9 @@ ERROR(forbidden_extended_escaping_string,none,
 ERROR(regex_literal_parsing_error,none,
       "%0", (StringRef))
 
+ERROR(prefix_slash_not_allowed,none,
+      "prefix slash not allowed", ())
+
 //------------------------------------------------------------------------------
 // MARK: Lexer diagnostics
 //------------------------------------------------------------------------------

include/swift/Parse/Lexer.h

@@ -531,6 +531,9 @@ class Lexer {
     void operator=(const SILBodyRAII&) = delete;
   };
 
+  /// Attempt to re-lex a regex literal with forward slashes `/.../`.
+  bool tryLexAsForwardSlashRegexLiteral(State S);
+
 private:
   /// Nul character meaning kind.
   enum class NulCharacterKind {

include/swift/Parse/Parser.h

@@ -559,6 +559,11 @@ class Parser {
     return f(backtrackScope);
   }
 
+  /// Discard the current token. This will avoid interface hashing or updating
+  /// the previous loc. Only should be used if you've completely re-lexed
+  /// a different token at that position.
+  SourceLoc discardToken();
+
   /// Consume a token that we created on the fly to correct the original token
   /// stream from lexer.
   void consumeExtraToken(Token K);
@@ -1723,6 +1728,7 @@ class Parser {
   ParserResult<Expr>
   parseExprPoundCodeCompletion(Optional<StmtKind> ParentKind);
 
+  UnresolvedDeclRefExpr *makeExprOperator(Token opToken);
   UnresolvedDeclRefExpr *parseExprOperator();
 
   void validateCollectionElement(ParserResult<Expr> element);

lib/Parse/Lexer.cpp

@@ -1997,6 +1997,62 @@ bool Lexer::tryLexRegexLiteral(const char *TokStart) {
   return true;
 }
 
+bool Lexer::tryLexAsForwardSlashRegexLiteral(State S) {
+  auto priorState = getStateForBeginningOfToken(NextToken, LeadingTrivia);
+
+  // Re-lex from the given state.
+  restoreState(S);
+
+  // While we restored state, that would have re-advanced the lexer. This is
+  // good in that it's filled in all the interesting properties of the token
+  // (trivia, is on new line, etc), but we need to rewind to re-lex the actual
+  // kind.
+  auto *TokStart = getBufferPtrForSourceLoc(NextToken.getLoc());
+  assert(*TokStart == '/');
+  CurPtr = TokStart + 1;
+
+  auto bail = [&]() -> bool {
+    restoreState(priorState);
+    return false;
+  };
+
+  // We need to ban these characters at the start of a regex to avoid ambiguity
+  // with unapplied operator references, e.g 'foo(/, /)'.
+  switch (*CurPtr) {
+  case ' ': case '\t': case ',': case ')':
+    return bail();
+  default:
+    break;
+  }
+
+  while (true) {
+    uint32_t CharValue = validateUTF8CharacterAndAdvance(CurPtr, BufferEnd);
+    if (CharValue == ~0U)
+      return bail();
+
+    // Regex literals cannot span multiple lines.
+    if (CharValue == '\n' || CharValue == '\r')
+      return bail();
+
+    if (CharValue == '\\' && *CurPtr == '/') {
+      // Skip escaped delimiter and advance.
+      CurPtr++;
+    } else if (CharValue == '/') {
+      // End of literal, stop.
+      break;
+    }
+  }
+  // We've ended on the opening of a comment, bail.
+  // TODO: We could treat such cases as postfix operators on a regex literal,
+  // but it seems more likely the user has written a comment and is in the
+  // middle of editing the text before it.
+  if (*CurPtr == '*' || *CurPtr == '/')
+    return bail();
+
+  formToken(tok::regex_literal, TokStart);
+  return true;
+}
+
 /// lexEscapedIdentifier:
 ///   identifier ::= '`' identifier '`'
 ///

lib/Parse/ParseDecl.cpp

@@ -8524,6 +8524,11 @@ Parser::parseDeclOperator(ParseDeclOptions Flags, DeclAttributes &Attributes) {
                                    Tok.getRawText().front() == '!'))
       diagnose(Tok, diag::postfix_operator_name_cannot_start_with_unwrap);
 
+  if (Attributes.hasAttribute<PrefixAttr>()) {
+    if (Tok.getText().contains("/"))
+      diagnose(Tok, diag::prefix_slash_not_allowed);
+  }
+
   // A common error is to try to define an operator with something in the
   // unicode plane considered to be an operator, or to try to define an
   // operator like "not".  Analyze and diagnose this specifically.

lib/Parse/ParseExpr.cpp

@@ -511,6 +511,31 @@ ParserResult<Expr> Parser::parseExprSequenceElement(Diag<> message,
 ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
   SyntaxParsingContext UnaryContext(SyntaxContext, SyntaxContextKind::Expr);
   UnresolvedDeclRefExpr *Operator;
+
+  // First check to see if we have the start of a regex literal /.../.
+  switch (Tok.getKind()) {
+  case tok::oper_prefix:
+  case tok::oper_binary_spaced:
+  case tok::oper_binary_unspaced: {
+    if (!Tok.getText().startswith("/"))
+      break;
+
+    // Try re-lex as a /.../ regex literal.
+    if (!L->tryLexAsForwardSlashRegexLiteral(getParserPosition().LS))
+      break;
+
+    // Discard the operator token, which will be replaced by the regex literal
+    // token.
+    discardToken();
+
+    assert(Tok.getText().startswith("/"));
+    assert(Tok.is(tok::regex_literal));
+    break;
+  }
+  default:
+    break;
+  }
+
   switch (Tok.getKind()) {
   default:
     // If the next token is not an operator, just parse this as expr-postfix.
@@ -532,16 +557,31 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
   case tok::backslash:
     return parseExprKeyPath();
 
-  case tok::oper_postfix:
+  case tok::oper_postfix: {
     // Postfix operators cannot start a subexpression, but can happen
     // syntactically because the operator may just follow whatever precedes this
     // expression (and that may not always be an expression).
     diagnose(Tok, diag::invalid_postfix_operator);
     Tok.setKind(tok::oper_prefix);
-    LLVM_FALLTHROUGH;
-  case tok::oper_prefix:
     Operator = parseExprOperator();
     break;
+  }
+  case tok::oper_prefix: {
+    // Check to see if we can split a prefix operator containing '/', e.g '!/',
+    // which might be a prefix operator on a regex literal.
+    auto slashIdx = Tok.getText().find("/");
+    if (slashIdx != StringRef::npos) {
+      auto prefix = Tok.getText().take_front(slashIdx);
+      if (!prefix.empty()) {
+        Operator = makeExprOperator({Tok.getKind(), prefix});
+        consumeStartingCharacterOfCurrentToken(Tok.getKind(), prefix.size());
+        break;
+      }
+      diagnose(Tok, diag::prefix_slash_not_allowed);
+    }
+    Operator = parseExprOperator();
+    break;
+  }
   case tok::oper_binary_spaced:
   case tok::oper_binary_unspaced: {
     // For recovery purposes, accept an oper_binary here.
@@ -860,19 +900,24 @@ static DeclRefKind getDeclRefKindForOperator(tok kind) {
   }
 }
 
-/// parseExprOperator - Parse an operator reference expression.  These
-/// are not "proper" expressions; they can only appear in binary/unary
-/// operators.
-UnresolvedDeclRefExpr *Parser::parseExprOperator() {
+UnresolvedDeclRefExpr *Parser::makeExprOperator(Token Tok) {
   assert(Tok.isAnyOperator());
   DeclRefKind refKind = getDeclRefKindForOperator(Tok.getKind());
   SourceLoc loc = Tok.getLoc();
   DeclNameRef name(Context.getIdentifier(Tok.getText()));
-  consumeToken();
   // Bypass local lookup.
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
 
+/// parseExprOperator - Parse an operator reference expression.  These
+/// are not "proper" expressions; they can only appear in binary/unary
+/// operators.
+UnresolvedDeclRefExpr *Parser::parseExprOperator() {
+  auto *op = makeExprOperator(Tok);
+  consumeToken();
+  return op;
+}
+
 /// parseExprSuper
 ///
 ///   expr-super:

lib/Parse/ParseRegex.cpp

@@ -32,13 +32,31 @@ using namespace swift::syntax;
 
 ParserResult<Expr> Parser::parseExprRegexLiteral() {
   assert(Tok.is(tok::regex_literal));
-  assert(regexLiteralParsingFn);
+
+  // Bail if '-enable-experimental-string-processing' is not enabled.
+  if (!Context.LangOpts.EnableExperimentalStringProcessing ||
+      !regexLiteralParsingFn) {
+    diagnose(Tok, diag::regex_literal_parsing_error,
+             "regex literal requires '-enable-experimental-string-processing'");
+    auto loc = consumeToken();
+    return makeParserResult(new (Context) ErrorExpr(loc));
+  }
 
   SyntaxParsingContext LocalContext(SyntaxContext,
                                     SyntaxKind::RegexLiteralExpr);
 
   auto regexText = Tok.getText();
 
+  // The Swift library doesn't know about `/.../` regexes, let's pretend it's
+  // `#/.../#` instead.
+  if (regexText[0] == '/') {
+    SmallString<32> scratch;
+    scratch.append("#");
+    scratch.append(regexText);
+    scratch.append("#");
+    regexText = Context.AllocateCopy(StringRef(scratch));
+  }
+
   // Let the Swift library parse the contents, returning an error, or null if
   // successful.
   // TODO: We need to be able to pass back a source location to emit the error

lib/Parse/Parser.cpp

@@ -579,13 +579,16 @@ const Token &Parser::peekToken() {
   return L->peekNextToken();
 }
 
-SourceLoc Parser::consumeTokenWithoutFeedingReceiver() {
-  SourceLoc Loc = Tok.getLoc();
+SourceLoc Parser::discardToken() {
   assert(Tok.isNot(tok::eof) && "Lexing past eof!");
+  SourceLoc Loc = Tok.getLoc();
+  L->lex(Tok, LeadingTrivia, TrailingTrivia);
+  return Loc;
+}
 
+SourceLoc Parser::consumeTokenWithoutFeedingReceiver() {
   recordTokenHash(Tok);
-
-  L->lex(Tok, LeadingTrivia, TrailingTrivia);
+  auto Loc = discardToken();
   PreviousLoc = Loc;
   return Loc;
 }

test/StringProcessing/Parse/forward-slash-regex-default.swift

@@ -0,0 +1,13 @@
+// RUN: %target-typecheck-verify-swift
+
+let _ = /x/ // expected-error {{regex literal requires '-enable-experimental-string-processing'}}
+
+prefix operator /  // expected-error {{prefix slash not allowed}}
+prefix operator ^/ // expected-error {{prefix slash not allowed}}
+
+_ = /x
+// expected-error@-1 {{prefix slash not allowed}}
+// expected-error@-2 {{'/' is not a prefix unary operator}}
+// expected-error@-3 {{cannot find 'x' in scope}}
+
+_ = !/x/ // expected-error {{regex literal requires '-enable-experimental-string-processing'}}