Refactor rhino code to work around internal Google

limitation on Java 8 use.
After this change Java 8 can be used anywhere except in
TokenStream and JSIdentifier.

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=188569116

src/com/google/javascript/jscomp/gwt/super/com/google/javascript/rhino/TokenUtil.java

@@ -33,8 +33,14 @@ private static class RegExp {
     native boolean test(int charcode);
   }
 
+  // TODO(moz): Support full range of valid characters.
+  private static final RegExp JS_IDENTIFIER_REGEX = new RegExp("^[a-zA-Z_$][\\w$]*$");
   private static final RegExp WHITE_SPACE_REGEX = new RegExp("\\s");
 
+  static boolean isJSIdentifier(String s) {
+    return JS_IDENTIFIER_REGEX.test(s);
+  }
+
   public static boolean isJSSpace(int c) {
     if (c <= 127) {
       return c == 0x20 || c == 0x9 || c == 0xC || c == 0xB;

src/com/google/javascript/rhino/TokenStream.java

@@ -51,10 +51,8 @@
  * It is based on the C source files jsscan.c and jsscan.h
  * in the jsref package.
  *
- *
- * IMPORTANT: As of 2018-03-09 it is still not possible to use Java 8 features in this file
- * due to limitations on some internal Google projects that depend on it.
  */
+
 public class TokenStream {
    public static boolean isKeyword(String name) {
         boolean id = false;
@@ -189,7 +187,7 @@ else if (c=='s') {
         return id;
     }
 
-  public static boolean isJSIdentifier(String s) {
-    return JSIdentifier.isJSIdentifier(s);
-  }
+    public static boolean isJSIdentifier(String s) {
+      return TokenUtil.isJSIdentifier(s);
+    }
 }

src/com/google/javascript/rhino/TokenUtil.java

@@ -56,6 +56,24 @@
  */
 @GwtIncompatible("Unsupported java.lang.Character fields")
 public class TokenUtil {
+  static boolean isJSIdentifier(String s) {
+    int length = s.length();
+
+    if (length == 0
+        || Character.isIdentifierIgnorable(s.charAt(0))
+        || !Character.isJavaIdentifierStart(s.charAt(0))) {
+      return false;
+    }
+
+    for (int i = 1; i < length; i++) {
+      if (Character.isIdentifierIgnorable(s.charAt(i))
+          || !Character.isJavaIdentifierPart(s.charAt(i))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   /* As defined in ECMA.  jsscan.c uses C isspace() (which allows
    * \v, I think.)  note that code in getChar() implicitly accepts
    * '\r' == \u000D as well.