Remove unused ScanFlagSuppressIdPid mode, and refactor Ident::Tk to replace HashTbl::TkFromNameLen

Author: curtisman

lib/Parser/Hash.cpp

@@ -157,8 +157,46 @@ uint HashTbl::CountAndVerifyItems(IdentPtr *buckets, uint bucketCount, uint mask
 }
 #endif
 
+
 #pragma warning(push)
-#pragma warning(disable:4740) // flow in or out of inline asm code suppresses global optimization
+#pragma warning(disable:4740)  // flow in or out of inline asm code suppresses global optimization
+
+// Decide if token is keyword by string matching -
+// This method is used during colorizing when scanner isn't interested in storing the actual id and does not care about conversion of escape sequences
+tokens Ident::TkFromNameLen(uint32 luHash, _In_reads_(cch) LPCOLESTR prgch, uint32 cch, bool isStrictMode, ushort * pgrfid, ushort * ptk)
+{
+    // look for a keyword
+    #include "kwds_sw.h"
+
+    #define KEYWORD(tk,f,prec2,nop2,prec1,nop1,name) \
+    LEqual_##name: \
+        if (cch == g_ssym_##name.cch && \
+                0 == memcmp(g_ssym_##name.sz, prgch, cch * sizeof(OLECHAR))) \
+        { \
+            if (f) \
+                *pgrfid |= f; \
+            *ptk = tk; \
+            return ((f & fidKwdRsvd) || (isStrictMode && (f & fidKwdFutRsvd))) ? tk : tkID; \
+        } \
+        goto LDefault;
+    #include "keywords.h"
+
+LDefault:
+    return tkID;
+}
+
+#pragma warning(pop)
+
+#if DBG
+tokens Ident::TkFromNameLen(_In_reads_(cch) LPCOLESTR prgch, uint32 cch, bool isStrictMode)
+{
+    uint32 luHash = CaseSensitiveComputeHash(prgch, prgch + cch);
+    ushort grfid;
+    ushort tk;
+    return TkFromNameLen(luHash, prgch, cch, isStrictMode, &grfid, &tk);
+}
+#endif
+
 tokens Ident::Tk(bool isStrictMode)
 {
     const tokens token = (tokens)m_tk;
@@ -168,22 +206,8 @@ tokens Ident::Tk(bool isStrictMode)
         const uint32 luHash = this->m_luHash;
         const LPCOLESTR prgch = Psz();
         const uint32 cch = Cch();
-        #include "kwds_sw.h"
-
-        #define KEYWORD(tk,f,prec2,nop2,prec1,nop1,name) \
-            LEqual_##name: \
-                if (cch == g_ssym_##name.cch && \
-                        0 == memcmp(g_ssym_##name.sz, prgch, cch * sizeof(OLECHAR))) \
-                { \
-                    if (f) \
-                        this->m_grfid |= f; \
-                    this->m_tk = tk; \
-                    return ((f & fidKwdRsvd) || (isStrictMode && (f & fidKwdFutRsvd))) ? tk : tkID; \
-                } \
-                goto LDefault;
-        #include "keywords.h"
-LDefault:
-        return tkID;
+
+        return TkFromNameLen(luHash, prgch, cch, isStrictMode, &this->m_grfid, &this->m_tk);
     }
     else if (token == tkNone || !(m_grfid & fidKwdRsvd))
     {
@@ -194,7 +218,6 @@ tokens Ident::Tk(bool isStrictMode)
     }
     return token;
 }
-#pragma warning(pop)
 
 void Ident::SetTk(tokens token, ushort grfid)
 {
@@ -423,33 +446,6 @@ bool HashTbl::Contains(_In_reads_(cch) LPCOLESTR prgch, int32 cch)
 
 #include "HashFunc.cpp"
 
-#pragma warning(push)
-#pragma warning(disable:4740)  // flow in or out of inline asm code suppresses global optimization
-
-// Decide if token is keyword by string matching -
-// This method is used during colorizing when scanner isn't interested in storing the actual id and does not care about conversion of escape sequences
-tokens HashTbl::TkFromNameLen(_In_reads_(cch) LPCOLESTR prgch, uint32 cch, bool isStrictMode)
-{
-    uint32 luHash = CaseSensitiveComputeHash(prgch, prgch + cch);
-
-    // look for a keyword
-#include "kwds_sw.h"
-
-    #define KEYWORD(tk,f,prec2,nop2,prec1,nop1,name) \
-        LEqual_##name: \
-            if (cch == g_ssym_##name.cch && \
-                    0 == memcmp(g_ssym_##name.sz, prgch, cch * sizeof(OLECHAR))) \
-            { \
-                return ((f & fidKwdRsvd) || (isStrictMode && (f & fidKwdFutRsvd))) ? tk : tkID; \
-            } \
-            goto LDefault;
-#include "keywords.h"
-
-LDefault:
-    return tkID;
-}
-
-#pragma warning(pop)
 
 __declspec(noreturn) void HashTbl::OutOfMemory()
 {

lib/Parser/Hash.h

@@ -322,6 +322,12 @@ struct Ident
 
     void SetIsModuleExport() { m_grfid |= fidModuleExport; }
     BOOL GetIsModuleExport() const { return m_grfid & fidModuleExport; }
+
+    static tokens TkFromNameLen(uint32 luHash, _In_reads_(cch) LPCOLESTR prgch, uint32 cch, bool isStrictMode, ushort * pgrfid, ushort * ptk);
+
+#if DBG
+    static tokens TkFromNameLen(_In_reads_(cch) LPCOLESTR prgch, uint32 cch, bool isStrictMode);
+#endif
 };
 
 
@@ -389,7 +395,6 @@ class HashTbl
 #endif
         );
 
-    tokens TkFromNameLen(_In_reads_(cch) LPCOLESTR prgch, uint32 cch, bool isStrictMode);
     NoReleaseAllocator* GetAllocator() {return &m_noReleaseAllocator;}
 
     bool Contains(_In_reads_(cch) LPCOLESTR prgch, int32 cch);

lib/Parser/Scan.cpp

@@ -465,26 +465,6 @@ tokens Scanner<EncodingPolicy>::ScanIdentifierContinue(bool identifyKwds, bool f
         return tkID;
     }
 
-    // During syntax coloring, scanner doesn't need to convert the escape sequence to get actual characters, it just needs the classification information
-    // So call up hashtables custom method to check if the string scanned is identifier or keyword.
-    // Do the same for deferred parsing, but use a custom method that only tokenizes JS keywords.
-    if ((m_DeferredParseFlags & ScanFlagSuppressIdPid) != 0)
-    {
-        m_ptoken->SetIdentifier(NULL);
-        if (!fHasEscape)
-        {
-            // If there are no escape, that the main scan loop would have found the keyword already
-            // So we can just assume it is an ID
-            DebugOnly(int32 cch = UnescapeToTempBuf(pchMin, p));
-            DebugOnly(tokens tk = m_phtbl->TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode()));
-            Assert(tk == tkID || (tk == tkYIELD && !this->YieldIsKeyword()) || (tk == tkAWAIT && !this->AwaitIsKeyword()));
-            return tkID;
-        }
-        int32 cch = UnescapeToTempBuf(pchMin, p);
-        tokens tk = m_phtbl->TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode());
-        return (!this->YieldIsKeyword() && tk == tkYIELD) || (!this->AwaitIsKeyword() && tk == tkAWAIT) ? tkID : tk;
-    }
-
     // UTF16 Scanner are only for syntax coloring, so it shouldn't come here.
     if (EncodingPolicy::MultiUnitEncoding && !fHasMultiChar && !fHasEscape)
     {
@@ -493,7 +473,7 @@ tokens Scanner<EncodingPolicy>::ScanIdentifierContinue(bool identifyKwds, bool f
         // If there are no escape, that the main scan loop would have found the keyword already
         // So we can just assume it is an ID
         DebugOnly(int32 cch = UnescapeToTempBuf(pchMin, p));
-        DebugOnly(tokens tk = m_phtbl->TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode()));
+        DebugOnly(tokens tk = Ident::TkFromNameLen(m_tempChBuf.m_prgch, cch, IsStrictMode()));
         Assert(tk == tkID || (tk == tkYIELD && !this->YieldIsKeyword()) || (tk == tkAWAIT && !this->AwaitIsKeyword()));
 
         m_ptoken->SetIdentifier(reinterpret_cast<const char *>(pchMin), (int32)(p - pchMin));
@@ -506,8 +486,8 @@ tokens Scanner<EncodingPolicy>::ScanIdentifierContinue(bool identifyKwds, bool f
     if (!fHasEscape)
     {
         // If it doesn't have escape, then Scan() should have taken care of keywords (except
-        // yield if this->YieldIsKeyword() is false, in which case yield is treated as an identifier, and except
-        // await if this->AwaitIsKeyword() is false, in which case await is treated as an identifier).
+        // yield if m_fYieldIsKeyword is false, in which case yield is treated as an identifier, and except
+        // await if m_fAwaitIsKeyword is false, in which case await is treated as an identifier).
         // We don't have to check if the name is reserved word and return it as an Identifier
         Assert(pid->Tk(IsStrictMode()) == tkID
             || (pid->Tk(IsStrictMode()) == tkYIELD && !this->YieldIsKeyword())

lib/Parser/Scan.h

@@ -318,7 +318,6 @@ enum ScanFlag
 {
     ScanFlagNone = 0,
     ScanFlagSuppressStrPid = 1,   // Force strings to always have pid
-    ScanFlagSuppressIdPid = 2     // Force identifiers to always have pid (currently unused)
 };
 
 typedef HRESULT (*CommentCallback)(void *data, OLECHAR firstChar, OLECHAR secondChar, bool containTypeDef, charcount_t min, charcount_t lim, bool adjacent, bool multiline, charcount_t startLine, charcount_t endLine);