JSON rewrite

lib/rouge/lexers/hocon.rb

@@ -11,33 +11,59 @@ class HOCON < JSON
       tag 'hocon'
       filenames '*.hocon'
 
-      prepend :root do
+      state :comments do
         # Comments
         rule %r(//.*?$), Comment::Single
         rule %r(#.*?$), Comment::Single
+      end
+
+      prepend :root do
+        mixin :comments
+      end
+
+      prepend :object do
+        # Keywords
+        rule %r/\b(?:include|url|file|classpath)\b/, Keyword
+      end
+
+      state :name do
+        rule %r/("(?:\"|[^"\n])*?")(\s*)([:=]|(?={))/ do
+          groups Name::Label, Text::Whitespace, Punctuation
+        end
+
+        rule %r/([-\w.]+)(\s*)([:=]|(?={))/ do
+          groups Name::Label, Text::Whitespace, Punctuation
+        end
+      end
+
+      state :value do
+        mixin :comments
+
+        rule %r/\n/, Text::Whitespace
+        rule %r/\s+/, Text::Whitespace
+
+        mixin :constants
 
         # Interpolation
         rule %r/[$][{][?]?/, Literal::String::Interpol, :interpolation
 
         # Strings
         rule %r/"""/, Literal::String::Double, :multiline_string
+        rule %r/"/, Str::Double, :string
 
-        # Keywords
-        rule %r/\b(?:include|url|file|classpath)\b/, Keyword
+        rule %r/\[/, Punctuation, :array
+        rule %r/{/, Punctuation, :object
 
         # Symbols (only those not handled by JSON)
         rule %r/[()=]/, Punctuation
 
-        # Keys
-        rule %r/([\w\-\.]+? *)([{:=]|\+=)/ do
-          groups Name::Attribute, Punctuation::Indicator
-        end
-
-        # Numbers (handle the case where we have multiple periods, ie. IP addresses)
-        rule %r/\d+\.(\d+\.?){3,}/, Literal #
-
         # Values
-        rule %r/[^\$\"{}\[\]:=,\+#`\^\?!@\*&]+?/, Literal
+        rule %r/[^$"{}\[\]:=,\+#`^?!@*&]+?/, Literal
+      end
+
+      state :interpolation do
+        rule %r/[\w\-\.]+?/, Name::Variable
+        rule %r/}/, Literal::String::Interpol, :pop!
       end
 
       prepend :string do
@@ -51,9 +77,9 @@ class HOCON < JSON
         rule %r/"""/, Literal::String::Double, :pop!
       end
 
-      state :interpolation do
-        rule %r/[\w\-\.]+?/, Name::Variable
-        rule %r/}/, Literal::String::Interpol, :pop!
+      prepend :constants do
+        # Numbers (handle the case where we have multiple periods, ie. IP addresses)
+        rule %r/\d+\.(\d+\.?){3,}/, Literal
       end
     end
   end

lib/rouge/lexers/json.rb

@@ -12,20 +12,58 @@ class JSON < RegexLexer
                 'application/hal+json', 'application/problem+json',
                 'application/schema+json'
 
+      state :whitespace do
+        rule %r/\s+/, Text::Whitespace
+      end
+
       state :root do
-        rule %r/\s+/m, Text::Whitespace
+        mixin :whitespace
+        rule %r/{/, Punctuation, :object
+        rule %r/\[/, Punctuation, :array
+
+        mixin :name
+        mixin :value
+      end
+
+      state :object do
+        mixin :whitespace
+        mixin :name
+        mixin :value
+        rule %r/}/, Punctuation, :pop!
+        rule %r/,/, Punctuation
+      end
+
+      state :name do
+        rule %r/("(?:\"|[^"\n])*?")(\s*)(:)/ do
+          groups Name::Label, Text::Whitespace, Punctuation
+        end
+      end
+
+      state :value do
+        mixin :whitespace
+        mixin :constants
         rule %r/"/, Str::Double, :string
-        rule %r/(?:true|false|null)\b/, Keyword::Constant
-        rule %r/[{},:\[\]]/, Punctuation
-        rule %r/-?(?:0|[1-9]\d*)\.\d+(?:e[+-]?\d+)?/i, Num::Float
-        rule %r/-?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer
+        rule %r/\[/, Punctuation, :array
+        rule %r/{/, Punctuation, :object
       end
 
       state :string do
         rule %r/[^\\"]+/, Str::Double
         rule %r/\\./, Str::Escape
         rule %r/"/, Str::Double, :pop!
       end
+
+      state :array do
+        mixin :value
+        rule %r/\]/, Punctuation, :pop!
+        rule %r/,/, Punctuation
+      end
+
+      state :constants do
+        rule %r/(?:true|false|null)/, Keyword::Constant
+        rule %r/-?(?:0|[1-9]\d*)\.\d+(?:e[+-]?\d+)?/i, Num::Float
+        rule %r/-?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer
+      end
     end
   end
 end

spec/lexers/javascript_spec.rb

@@ -18,17 +18,10 @@
 
     it 'guesses by filename' do
       assert_guess :filename => 'foo.js'
-      assert_guess Rouge::Lexers::JSON, :filename => 'foo.json'
-      assert_guess Rouge::Lexers::JSON, :filename => 'Pipfile.lock'
     end
 
     it 'guesses by mimetype' do
       assert_guess :mimetype => 'text/javascript'
-      assert_guess Rouge::Lexers::JSON, :mimetype => 'application/json'
-      assert_guess Rouge::Lexers::JSON, :mimetype => 'application/vnd.api+json'
-      assert_guess Rouge::Lexers::JSON, :mimetype => 'application/hal+json'
-      assert_guess Rouge::Lexers::JSON, :mimetype => 'application/problem+json'
-      assert_guess Rouge::Lexers::JSON, :mimetype => 'application/schema+json'
     end
 
     it 'guesses by source' do

spec/lexers/json_spec.rb

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Lexers::JSON do
+  let(:subject) { Rouge::Lexers::JSON.new }
+
+  describe 'guessing' do
+    include Support::Guessing
+
+    it 'guesses by filename' do
+      assert_guess :filename => 'foo.json'
+      assert_guess :filename => 'Pipfile.lock'
+    end
+
+    it 'guesses by mimetype' do
+      assert_guess :mimetype => 'application/json'
+      assert_guess :mimetype => 'application/vnd.api+json'
+      assert_guess :mimetype => 'application/hal+json'
+      assert_guess :mimetype => 'application/problem+json'
+      assert_guess :mimetype => 'application/schema+json'
+    end
+
+    it 'guesses by source' do
+      assert_guess :mimetype => 'application/json', :source => '{"name": "value"}'
+    end
+  end
+end

spec/visual/samples/json

@@ -1,25 +1,142 @@
 {
-    "firstName": "John",
-    "lastName": "Smith",
-    "age": 25,
-    "address": {
-        "streetAddress": "21 2nd Street",
-        "city": "New York",
-        "state": "NY",
-        "postalCode": "10021"
-    },
-    "phoneNumber": [
-        {
-            "type": "home",
-            "number": "212 555-1234"
-        },
-        {
-            "type": "fax",
-            "number": "646 555-4567"
-        }
-    ],
-    "errors": {
-        "object": { "a", "b" }
-    },
-    "emptyObject": {}
+  "problems": [
+      {"question": "how many miles?", "answer": "6"},
+      {"question": "how many kilometers?", "answer": "10"}
+  ],
+  "testing \" something \"": 1,
+  "firstName": "John",
+  "lastName": "Smith",
+  "age": 25,
+  "address": {
+      "streetAddress": "21 2nd Street",
+      "city": "New York",
+      "state": "NY",
+      "postalCode": "10021"
+  },
+  "strings": ["one", "two", "three", "four"],
+  "nested1": [[1], [2], [3]],
+  "numbers": [1,  2,   3, 4],
+  "phoneNumber": [
+      {
+          "type": "home",
+          "number": "212 555-1234"
+      },
+      {
+          "type": "fax",
+          "number": "646 555-4567"
+      }
+  ],
+  "life_is_good": true,
+  "life_is_bad": 10e-4,
+  "errors": {
+      "object": { "a": 1, "b": 2,
+      "c": "some string" }
+  },
+  "emptyObject": {},
+  "color": {
+      "rgba": [0,255,0,1],
+      "string": "black",
+      "hex": "#0F0"
+    }
 }
+
+"this": { "is": "not syntactically correct JSON but is here for backwards compatibility" }
+true
+100
+null
+"a": "test"
+"string"
+
+{ "statement": "SELECT (CASE WHEN (SELECT count(*) FROM \"ci_builds\" WHERE 
+               \"ci_builds\".\"commit_id\" = 77 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\"))=0 THEN
+               NULL WHEN (SELECT count(*) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\"))=(SELECT
+               count(*) FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" =
+               77 AND \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id)
+               FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP
+               BY \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'skipped') THEN 'skipped' WHEN
+               (SELECT count(*) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\"))=(SELECT
+               count(*) FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" =
+               77 AND \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id)
+               FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP
+               BY \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'success')+(SELECT count(*) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 AND
+               \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"allow_failure\" = 't' AND
+               \"ci_builds\".\"status\" IN ('failed', 'canceled'))+(SELECT
+               count(*) FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" =
+               77 AND \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id)
+               FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP
+               BY \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'skipped') THEN 'success' WHEN
+               (SELECT count(*) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\"))=(SELECT
+               count(*) FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" =
+               77 AND \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id)
+               FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP
+               BY \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'pending')+(SELECT count(*) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 AND
+               \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'skipped') THEN 'pending' WHEN
+               (SELECT count(*) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\"))=(SELECT
+               count(*) FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" =
+               77 AND \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id)
+               FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP
+               BY \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'canceled')+(SELECT count(*) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 AND
+               \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'success')+(SELECT count(*) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 AND
+               \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"allow_failure\" = 't' AND
+               \"ci_builds\".\"status\" IN ('failed', 'canceled'))+(SELECT
+               count(*) FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" =
+               77 AND \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id)
+               FROM \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP
+               BY \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'skipped') THEN 'canceled' WHEN
+               (SELECT count(*) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'running')+(SELECT count(*) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 AND
+               \"ci_builds\".\"id\" IN (SELECT max(\"ci_builds\".id) FROM
+               \"ci_builds\" WHERE \"ci_builds\".\"commit_id\" = 77 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\") AND
+               \"ci_builds\".\"status\" = 'pending')>0 THEN 'running' ELSE
+               'failed' END) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = $1 AND \"ci_builds\".\"id\" IN
+               (SELECT max(\"ci_builds\".id) FROM \"ci_builds\" WHERE
+               \"ci_builds\".\"commit_id\" = $2 GROUP BY
+               \"ci_builds\".\"name\", \"ci_builds\".\"commit_id\")" }