Support custom token types

This change makes it possible to implement custom token types, which is
really useful when building specialized grammars that use non-standard
token types.

Sources/Splash/Output/HTMLOutputFormat.swift

@@ -33,7 +33,7 @@ public extension HTMLOutputFormat {
         }
 
         public mutating func addToken(_ token: String, ofType type: TokenType) {
-            html.append("<span class=\"\(classPrefix)\(type.rawValue)\">\(token.escaped)</span>")
+            html.append("<span class=\"\(classPrefix)\(type.string)\">\(token.escaped)</span>")
         }
 
         public mutating func addPlainText(_ text: String) {

Sources/Splash/Tokenizing/TokenType.swift

@@ -7,7 +7,7 @@
 import Foundation
 
 /// Enum defining the possible types of tokens that can be highlighted
-public enum TokenType: String, Equatable {
+public enum TokenType: Hashable {
     /// A keyword, such as `if`, `class` or `let`
     case keyword
     /// A token that is part of a string literal
@@ -26,4 +26,17 @@ public enum TokenType: String, Equatable {
     case dotAccess
     /// A preprocessing symbol, such as `#if` or `@available`
     case preprocessing
+    /// A custom token type, containing an arbitrary string
+    case custom(String)
+}
+
+public extension TokenType {
+    /// Return a string value representing the token type
+    var string: String {
+        if case .custom(let type) = self {
+            return type
+        }
+
+        return "\(self)"
+    }
 }

Sources/SplashTokenizer/TokenizerOutputFormat.swift

@@ -18,7 +18,7 @@ extension TokenizerOutputFormat {
         private var components = [String]()
 
         mutating func addToken(_ token: String, ofType type: TokenType) {
-            components.append("\(type.rawValue.capitalized) token: \(token)")
+            components.append("\(type.string.capitalized) token: \(token)")
         }
 
         mutating func addPlainText(_ text: String) {

Tests/SplashTests/Tests/TokenTypeTests.swift

@@ -0,0 +1,32 @@
+/**
+ *  Splash
+ *  Copyright (c) John Sundell 2018
+ *  MIT license - see LICENSE.md
+ */
+
+import Foundation
+import XCTest
+import Splash
+
+final class TokenTypeTests: SplashTestCase {
+    func testConvertingToString() {
+        let standardType = TokenType.comment
+        XCTAssertEqual(standardType.string, "comment")
+
+        let customType = TokenType.custom("MyCustomType")
+        XCTAssertEqual(customType.string, "MyCustomType")
+    }
+
+    func testAllTestsRunOnLinux() {
+        XCTAssertTrue(TestCaseVerifier.verifyLinuxTests((type(of: self)).allTests))
+    }
+}
+
+extension TokenTypeTests {
+    static var allTests: [(String, TestClosure<TokenTypeTests>)] {
+        return [
+            ("testConvertingToString", testConvertingToString)
+        ]
+    }
+}
+