Add Unicode symbol characters to operator charset.

Remove '@' from the operator character set, but add the math, symbol, punctuation, arrow, and line- and box-drawing characters. Also allow operators to contain (but not start with) combining characters--this should be safe, because identifiers can't start with combining characters either, and we don't allow them anywhere else.

Swift SVN r5019

Author: jckarter

docs/ABI.rst

@@ -171,6 +171,7 @@ there's no ambiguity with the run-length.
 ::
 
   identifier ::= 'X' natural identifier-start-char identifier-char*
+  identifier ::= 'X' 'o' operator-fixity natural identifier-char*
 
 Identifiers that contain non-ASCII characters are encoded using the Punycode
 algorithm specified in RFC 3492, with the modifications that ``_`` is used
@@ -181,6 +182,14 @@ encoded string itself. For example, the identifier ``vergüenza`` is mangled
 to ``X12vergenza_JFa``. (The encoding in standard Punycode would be
 ``vergenza-95a``)
 
+Operators that contain non-ASCII characters are mangled by first mapping the
+ASCII operator characters to letters as for pure ASCII operator names, then
+Punycode-encoding the substituted string. The mangling then consists of
+``Xo`` followed by the fixity, run length of the encoded string, and the encoded
+string itself. For example, the infix operator ``«+»`` is mangled to
+``Xoi7p_qcaDc`` (``p_qcaDc`` being the encoding of the substituted
+string ``«p»``).
+
 ::
 
   substitution ::= 'S' index

include/swift/AST/Identifier.h

@@ -20,6 +20,7 @@
 #include "swift/Basic/LLVM.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ConvertUTF.h"
 #include <cstring>
 
 namespace llvm {
@@ -67,15 +68,58 @@ class Identifier {
 
   /// isOperator - Return true if this identifier is an operator, false if it is
   /// a normal identifier.
+  /// FIXME: We should maybe cache this.
   bool isOperator() const {
-    return !empty() && isOperatorChar(Pointer[0]);
+    if (empty())
+      return false;
+    if ((unsigned char)Pointer[0] < 0x80)
+      return isOperatorStartCodePoint((unsigned char)Pointer[0]);
+    
+    StringRef data = str();
+    auto *s = reinterpret_cast<UTF8 const *>(data.begin()),
+       *end = reinterpret_cast<UTF8 const *>(data.end());
+    UTF32 codePoint;
+    ConversionResult res = llvm::convertUTF8Sequence(&s, end, &codePoint,
+                                                     strictConversion);
+    assert(res == conversionOK && "invalid UTF-8 in identifier?!");
+    return !empty() && isOperatorStartCodePoint(codePoint);
   }
 
-  /// isOperatorChar - Return true if the specified character is a
-  /// valid part of an operator.
-  static bool isOperatorChar(char C) {
-    static const char OpChars[] = "@/=-+*%<>!&|^~.";
-    return memchr(OpChars, C, sizeof(OpChars) - 1) != 0;
+  /// isOperatorStartCodePoint - Return true if the specified code point is a
+  /// valid start of an operator.
+  static bool isOperatorStartCodePoint(uint32_t C) {
+    // ASCII operator chars.
+    static const char OpChars[] = "/=-+*%<>!&|^~.";
+    if (C < 0x80)
+      return memchr(OpChars, C, sizeof(OpChars) - 1) != 0;
+    
+    // Unicode math, symbol, arrow, dingbat, and line/box drawing chars.
+    return (C >= 0x00A1 && C <= 0x00A7)
+        || C == 0x00A9 || C == 0x00AB || C == 0x00AC || C == 0x00AE
+        || C == 0x00B0 || C == 0x00B1 || C == 0x00B6 || C == 0x00BB
+        || C == 0x00BF || C == 0x00D7 || C == 0x00F7
+        || C == 0x2016 || C == 0x2017 || (C >= 0x2020 && C <= 0x2027)
+        || (C >= 0x2030 && C <= 0x203E) || (C >= 0x2041 && C <= 0x2053)
+        || (C >= 0x2055 && C <= 0x205E) || (C >= 0x2190 && C <= 0x23FF)
+        || (C >= 0x2500 && C <= 0x2775) || (C >= 0x2794 && C <= 0x2BFF)
+        || (C >= 0x2E00 && C <= 0x2E7F) || (C >= 0x3001 && C <= 0x3003)
+        || (C >= 0x3008 && C <= 0x3030);
+  }
+  
+  /// isOperatorContinuationCodePoint - Return true if the specified code point
+  /// is a valid operator code point.
+  static bool isOperatorContinuationCodePoint(uint32_t C) {
+    // '.' is a special case. It can only appear in '..'.
+    if (C == '.')
+      return false;
+    if (isOperatorStartCodePoint(C))
+      return true;
+
+    // Unicode combining characters.
+    return (C >= 0x0300 && C <= 0x036F)
+        || (C >= 0x1DC0 && C <= 0x1DFF)
+        || (C >= 0x20D0 && C <= 0x20FF)
+        || (C >= 0xFE20 && C <= 0xFE2F);
   }
 
   void *getAsOpaquePointer() const { return (void *)Pointer; }

lib/Parse/Lexer.cpp

@@ -376,7 +376,7 @@ static bool isValidIdentifierStartCodePoint(uint32_t c) {
 
   // N1518: Recommendations for extended identifier characters for C and C++
   // Proposed Annex X.2: Ranges of characters disallowed initially
-  if ((c >= 0x0300 && c <= 0x36F)
+  if ((c >= 0x0300 && c <= 0x036F)
       || (c >= 0x1DC0 && c <= 0x1DFF)
       || (c >= 0x20D0 && c <= 0x20FF)
       || (c >= 0xFE20 && c <= 0xFE2F))
@@ -385,30 +385,39 @@ static bool isValidIdentifierStartCodePoint(uint32_t c) {
   return true;
 }
 
-static bool advanceIfValidContinuationOfIdentifier(char const *&ptr,
-                                                   char const *end) {
+static bool advanceIf(char const *&ptr,
+                      char const *end,
+                      bool (*predicate)(uint32_t)) {
   char const *next = ptr;
   uint32_t c = validateUTF8CharacterAndAdvance(next, end);
   if (c == ~0U)
     return false;
-  if (isValidIdentifierContinuationCodePoint(c)) {
+  if (predicate(c)) {
     ptr = next;
     return true;
   }
   return false;
+
 }
 
 static bool advanceIfValidStartOfIdentifier(char const *&ptr,
                                             char const *end) {
-  char const *next = ptr;
-  uint32_t c = validateUTF8CharacterAndAdvance(next, end);
-  if (c == ~0U)
-    return false;
-  if (isValidIdentifierStartCodePoint(c)) {
-    ptr = next;
-    return true;
-  }
-  return false;
+  return advanceIf(ptr, end, isValidIdentifierStartCodePoint);
+}
+
+static bool advanceIfValidContinuationOfIdentifier(char const *&ptr,
+                                                   char const *end) {
+  return advanceIf(ptr, end, isValidIdentifierContinuationCodePoint);
+}
+
+static bool advanceIfValidStartOfOperator(char const *&ptr,
+                                          char const *end) {
+  return advanceIf(ptr, end, Identifier::isOperatorStartCodePoint);
+}
+
+static bool advanceIfValidContinuationOfOperator(char const *&ptr,
+                                                 char const *end) {
+  return advanceIf(ptr, end, Identifier::isOperatorContinuationCodePoint);
 }
 
 /// isIdentifier - Checks whether a string matches the identifier regex.
@@ -531,8 +540,12 @@ void Lexer::lexOperatorIdentifier() {
       return formToken(tok::unknown, TokStart);
     }
   } else {
-    while (Identifier::isOperatorChar(*CurPtr) && *CurPtr != '.')
-      ++CurPtr;
+    CurPtr = TokStart;
+    bool didStart = advanceIfValidStartOfOperator(CurPtr, BufferEnd);
+    assert(didStart && "unexpected operator start");
+    (void) didStart;
+    
+    while (advanceIfValidContinuationOfOperator(CurPtr, BufferEnd));
   }
 
   // Decide between the binary, prefix, and postfix cases.
@@ -1150,6 +1163,9 @@ void Lexer::lexImpl() {
     if (advanceIfValidStartOfIdentifier(tmp, BufferEnd))
       return lexIdentifier();
 
+    if (advanceIfValidStartOfOperator(tmp, BufferEnd))
+      return lexOperatorIdentifier();
+    
     if (advanceIfValidContinuationOfIdentifier(tmp, BufferEnd)) {
       // If this is a valid identifier continuation, but not a valid identifier
       // start, attempt to recover by eating more continuation characters.

lib/SIL/Mangle.cpp

@@ -51,7 +51,8 @@ static char mangleOperatorChar(char op) {
   case '~': return 't'; // 'tilde'
   case '^': return 'x'; // 'xor'
   case '.': return 'z'; // 'zperiod' (the z is silent)
-  default: llvm_unreachable("bad identifier character");
+  default:
+    return op;
   }
 }
 
@@ -88,21 +89,21 @@ void Mangler::mangleIdentifier(Identifier ident, OperatorFixity fixity) {
   StringRef str = ident.str();
   assert(!str.empty() && "mangling an empty identifier!");
 
+  // If the identifier contains non-ASCII character, we mangle with an initial
+  // X and Punycode the identifier string.
+  llvm::SmallString<32> punycodeBuf;
+
+  if (isNonAscii(str)) {
+    Buffer << 'X';
+    Punycode::encodePunycode(str, punycodeBuf);
+    str = punycodeBuf;
+  }
+  
   // Mangle normal identifiers as
   //   count identifier-char+
   // where the count is the number of characters in the identifier,
   // and where individual identifier characters represent themselves.
   if (!ident.isOperator()) {
-    // If the identifier contains non-ASCII characters, Punycode it, and mangle
-    // the encoded string as:
-    //   'X' count identifier-char+
-    if (isNonAscii(str)) {
-      llvm::SmallString<32> encoded;
-      Punycode::encodePunycode(str, encoded);
-      Buffer << 'X' << encoded.size() << encoded;
-      return;
-    }
-      
     Buffer << str.size() << str;
     return;
   }
@@ -129,9 +130,10 @@ void Mangler::mangleIdentifier(Identifier ident, OperatorFixity fixity) {
     break;
   }
 
+  // Mangle ASCII operators directly.
   Buffer << str.size();
-  for (unsigned i = 0, e = str.size(); i != e; ++i) {
-    Buffer << mangleOperatorChar(str[i]);
+  for (char c : str) {
+    Buffer << mangleOperatorChar(c);
   }
 }
 

tools/swift/Completion.cpp

@@ -224,16 +224,17 @@ static CompletionContext getCompletionContext(DeclContext *dc,
   } else if (prefix.back() == '.') {
     // Try to figure out a qualified lookup context from the expression
     // preceding the dot.
+    // FIXME: Unicode.
     StringRef beforeDot = prefix.slice(0, prefix.size() - 1);
     prefix = StringRef();
     return getCompletionContextFromDotExpression(TU, dc, beforeDot);
-  } else if (Identifier::isOperatorChar(prefix.back())) {
+  } else if (Identifier::isOperatorStartCodePoint(prefix.back())) {
     // If the character preceding us looks like an operator character,
     // walk backward to find as much of an operator name as we can.
     for (char const *p = prefix.end(), *end = p, *begin = prefix.begin();
          p != begin;
          --p) {
-      if (!Identifier::isOperatorChar(*(p-1))) {
+      if (!Identifier::isOperatorStartCodePoint(*(p-1))) {
         prefix = StringRef(p, end - p);
         return CompletionContext(dc, tuEndLoc);
       }

tools/swift/Immediate.cpp

@@ -555,8 +555,9 @@ class REPLInput {
         ++p;
       }
       while (isspace(*--p) && p >= s);
-      if (Identifier::isOperatorChar(*p)) {
-        while (Identifier::isOperatorChar(*p) && --p >= s);
+      // FIXME: Unicode operators
+      if (Identifier::isOperatorStartCodePoint(*p)) {
+        while (Identifier::isOperatorContinuationCodePoint(*p) && --p >= s);
         if (*p == ' ' || *p == '\t')
           UnfinishedInfixExpr = true;
       }