Add opEquals to Token

This allows to test two tokens for equality. It will include the kind
of token (`value`) and the contents of the token
(where applicable). The location will not be included in the equality.

src/dmd/root/ctfloat.d

@@ -41,6 +41,8 @@ extern (C++) struct CTFloat
   @nogc:
   @safe:
 
+    alias isNaN = .isNaN;
+
     version (GNU)
         enum yl2x_supported = false;
     else
@@ -140,12 +142,6 @@ extern (C++) struct CTFloat
         return calcHash(cast(ubyte*) &a, sz);
     }
 
-    pure
-    static bool isNaN(real_t r)
-    {
-        return !(r == r);
-    }
-
     pure @trusted
     static bool isSNaN(real_t r)
     {
@@ -230,3 +226,8 @@ extern (C++) struct CTFloat
         half = real_t(0.5);
     }
 }
+
+bool isNaN(real_t r) pure @nogc @safe
+{
+    return !(r == r);
+}

src/dmd/tokens.d

@@ -709,6 +709,89 @@ extern (C++) struct Token
 
 nothrow:
 
+    /**
+     * Tests the receiver and the given value for equality.
+     *
+     * Two tokens are considered equal if they both are of the same kind
+     * (`value`) and has the same contents (where applicable). The location is
+     * not part of the equality test.
+     *
+     * Params:
+     *  rhs = the other value to test for equality
+     *
+     * Returns: `true` if the values are equal
+     */
+    bool opEquals()(auto ref const Token rhs) const pure nothrow @nogc
+    {
+        import dmd.root.ctfloat : isNaN;
+
+        if (value != rhs.value)
+            return false;
+
+        with (TOK) switch (value)
+        {
+            case int32Literal,
+                uns32Literal,
+                int64Literal,
+                uns64Literal,
+                charLiteral,
+                wcharLiteral,
+                dcharLiteral:
+                // unsvalue is used for signed integers and characters by the lexer
+                return unsvalue == rhs.unsvalue;
+
+            case int128Literal, uns128Literal:
+                // the lexer doesn't handle these
+                assert(false, "not implemented");
+
+            case float32Literal,
+                float64Literal,
+                float80Literal,
+                imaginary32Literal,
+                imaginary64Literal,
+                imaginary80Literal:
+                    return floatvalue == rhs.floatvalue ||
+                        floatvalue.isNaN && rhs.floatvalue.isNaN;
+
+            case string_, hexadecimalString:
+                return postfix == rhs.postfix &&
+                    ustring[0 .. len] == rhs.ustring[0 .. rhs.len];
+
+            case identifier,
+                enum_,
+                struct_,
+                import_,
+                wchar_,
+                dchar_,
+                bool_,
+                char_,
+                int8,
+                uns8,
+                int16,
+                uns16,
+                int32,
+                uns32,
+                int64,
+                uns64,
+                int128,
+                uns128,
+                float32,
+                float64,
+                float80,
+                imaginary32,
+                imaginary64,
+                imaginary80,
+                complex32,
+                complex64,
+                complex80,
+                void_:
+                    return ident is rhs.ident;
+
+            default:
+                return true;
+        }
+    }
+
     shared static this()
     {
         Identifier.initTable();

test/tools/unit_test_runner.d

@@ -228,6 +228,7 @@ void writeCmdfile(string path, string runnerPath, string outputPath,
     const string[] testFiles)
 {
     auto flags = [
+        "-debug",
         "-version=NoBackend",
         "-version=GC",
         "-version=NoMain",