added some tests for get_bytes and get_size, bump version 0.1.3

CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.1.3
+
+* Enhancements
+    * added support for size unit format
+    * added more tests 
+    * extended the documentation 
+
+* Bugfixes
+    * The `Hocon.Tokenizer` handles the variable byte length of UTF-8 strings correctly.
+    * Escaping unicodes in quoted string is now supported.
+
 ## 0.1.2
 
 * Enhancements

README.md

@@ -107,4 +107,4 @@ https://github.com/lightbend/config/blob/master/HOCON.md
 - [ ] allow URL for included files
 - [ ] duration unit format
 - [ ] period unit format
-- [ ] size unit format
+- [x] size unit format

lib/hocon.ex

@@ -1,32 +1,7 @@
 defmodule Hocon do
   @moduledoc"""
 
-  This module paresed and decodes a hocon configuration string.
-
-  ## [specification](https://github.com/lightbend/config/blob/master/HOCON.md) coverages:
-
-  - [x] parsing JSON
-  - [x] comments
-  - [x] omit root braces
-  - [x] key-value separator
-  - [x] commas are optional if newline is present
-  - [x] whitespace
-  - [x] duplicate keys and object merging
-  - [x] unquoted strings
-  - [x] multi-line strings
-  - [x] value concatenation
-  - [x] object concatenation
-  - [x] array concatenation
-  - [x] path expressions
-  - [x] path as keys
-  - [x] substitutions
-  - [ ] includes
-  - [x] conversion of numerically-indexed objects to arrays
-  - [ ] allow URL for included files
-  - [ ] duration unit format
-  - [ ] period unit format
-  - [x] size unit format
-
+  This module paresed and decodes a [hocon](https://github.com/lightbend/config/blob/master/HOCON.md) configuration string.
 
   ## Example
 
@@ -40,13 +15,13 @@ defmodule Hocon do
 
   The Parser returns a map, because in Elixir it is a common use case to use pattern matching on maps to
   extract specific values and keys. Therefore the `Hocon.decode/2` function returns a map. To support
-  interpreting a value with some family of units, you can call some conversion functions like `get_bytes/1`.
+  interpreting a value with some family of units, you can call some conversion functions like `as_bytes/1`.
 
   ## Example
 
        iex> conf = ~s(limit : "512KB")
        iex> {:ok, %{"limit" => limit}} = Hocon.decode(conf)
-       iex> Hocon.get_bytes(limit)
+       iex> Hocon.as_bytes(limit)
        524288
 
   """
@@ -137,59 +112,125 @@ defmodule Hocon do
   in case of errors.
   """
   def decode!(string, opts \\ []) do
-    with {:ok, result} <- Parser.decode(string, opts) do
-      result
+    Parser.decode!(string, opts)
+  end
+
+  @doc """
+  Returns a value for the `keypath` from a map or a successfull parse HOCON string.
+
+  ## Example
+      iex> conf = Hocon.decode!(~s(a { b { c : "10kb" } }))
+      %{"a" => %{"b" => %{"c" => "10kb"}}}
+      iex> Hocon.get(conf, "a.b.c")
+      "10kb"
+      iex> Hocon.get(conf, "a.b.d")
+      nil
+      iex> Hocon.get(conf, "a.b.d", "1kb")
+      "1kb"
+  """
+  def get(root, keypath, default \\ nil) do
+    keypath = keypath
+              |> String.split(".")
+              |> Enum.map(fn str -> String.trim(str) end)
+    case get_in(root, keypath) do
+       nil -> default
+        other -> other
+    end
+  end
+
+  @doc """
+  Same a `get/3` but the value is interpreted like a number by using the power of 2.
+
+  ## Example
+      iex> conf = Hocon.decode!(~s(a { b { c : "10kb" } }))
+      %{"a" => %{"b" => %{"c" => "10kb"}}}
+      iex> Hocon.get_bytes(conf, "a.b.c")
+      10240
+      iex> Hocon.get_bytes(conf, "a.b.d")
+      nil
+      iex> Hocon.get_bytes(conf, "a.b.d", 1024)
+      1024
+  """
+  def get_bytes(root, keypath, default \\ nil) do
+    keypath = keypath
+              |> String.split(".")
+              |> Enum.map(fn str -> String.trim(str) end)
+    case get_in(root, keypath) do
+      nil -> default
+      other -> as_bytes(other)
     end
   end
 
+  @doc """
+  Same a `get/3` but the value is interpreted like a number by using the power of 10.
+
+  ## Example
+      iex> conf = Hocon.decode!(~s(a { b { c : "10kb" } }))
+      %{"a" => %{"b" => %{"c" => "10kb"}}}
+      iex> Hocon.get_bytes(conf, "a.b.c")
+      10240
+      iex> Hocon.get_bytes(conf, "a.b.d")
+      nil
+      iex> Hocon.get_bytes(conf, "a.b.d", 1024)
+      1024
+  """
+  def get_size(root, keypath, default \\ nil) do
+    keypath = keypath
+              |> String.split(".")
+              |> Enum.map(fn str -> String.trim(str) end)
+    case get_in(root, keypath) do
+      nil -> default
+      other -> as_size(other)
+    end
+  end
 
   @doc """
   Returns the size of the `string` by using the power of 2.
 
   ## Example
-      iex> Hocon.get_bytes("512kb")
+      iex> Hocon.as_bytes("512kb")
       524288
-      iex> Hocon.get_bytes("125 gigabytes")
+      iex> Hocon.as_bytes("125 gigabytes")
       134217728000
   """
-  def get_bytes(value) when is_number(value), do: value
-  def get_bytes(string) when is_binary(string) do
-    get_bytes(Regex.named_captures(~r/(?<value>\d+)(\W)?(?<unit>[[:alpha:]]+)?/, String.downcase(string)))
+  def as_bytes(value) when is_number(value), do: value
+  def as_bytes(string) when is_binary(string) do
+    as_bytes(Regex.named_captures(~r/(?<value>\d+)(\W)?(?<unit>[[:alpha:]]+)?/, String.downcase(string)))
   end
-  def get_bytes(%{"unit" => "", "value" => value}), do: parse_integer(value, 1)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(b byte bytes), do: parse_integer(value, 1)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(k kb kilobyte kilobytes), do: parse_integer(value, @kb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(m mb megabyte megabytes), do: parse_integer(value, @mb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(g gb gigabyte gigabytes), do: parse_integer(value, @gb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(t tb terabyte terabytes), do: parse_integer(value, @tb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(p pb petabyte petabytes), do: parse_integer(value, @pb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(e eb exabyte exabytes), do: parse_integer(value, @eb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(z zb zettabyte zettabytes), do: parse_integer(value, @zb)
-  def get_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(y yb yottabyte yottabytes), do: parse_integer(value, @yb)
+  def as_bytes(%{"unit" => "", "value" => value}), do: parse_integer(value, 1)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(b byte bytes), do: parse_integer(value, 1)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(k kb kilobyte kilobytes), do: parse_integer(value, @kb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(m mb megabyte megabytes), do: parse_integer(value, @mb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(g gb gigabyte gigabytes), do: parse_integer(value, @gb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(t tb terabyte terabytes), do: parse_integer(value, @tb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(p pb petabyte petabytes), do: parse_integer(value, @pb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(e eb exabyte exabytes), do: parse_integer(value, @eb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(z zb zettabyte zettabytes), do: parse_integer(value, @zb)
+  def as_bytes(%{"unit" => unit, "value" => value}) when unit in ~w(y yb yottabyte yottabytes), do: parse_integer(value, @yb)
 
   @doc """
   Returns the size of the `string` by using the power of 10.
 
   ## Example
-      iex> Hocon.get_size("512kb")
+      iex> Hocon.as_size("512kb")
       512000
-      iex> Hocon.get_size("125 gigabytes")
+      iex> Hocon.as_size("125 gigabytes")
       125000000000
   """
-  def get_size(value) when is_number(value), do: value
-  def get_size(string) when is_binary(string) do
-    get_size(Regex.named_captures(~r/(?<value>\d+)(\W)?(?<unit>[[:alpha:]]+)?/, String.downcase(string)))
+  def as_size(value) when is_number(value), do: value
+  def as_size(string) when is_binary(string) do
+    as_size(Regex.named_captures(~r/(?<value>\d+)(\W)?(?<unit>[[:alpha:]]+)?/, String.downcase(string)))
   end
-  def get_size(%{"unit" => "", "value" => value}), do: parse_integer(value, 1)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(b byte bytes), do: parse_integer(value, 1)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(k kb kilobyte kilobytes), do: parse_integer(value, @kb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(m mb megabyte megabytes), do: parse_integer(value, @mb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(g gb gigabyte gigabytes), do: parse_integer(value, @gb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(t tb terabyte terabytes), do: parse_integer(value, @tb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(p pb petabyte petabytes), do: parse_integer(value, @pb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(e eb exabyte exabytes), do: parse_integer(value, @eb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(z zb zettabyte zettabytes), do: parse_integer(value, @zb_10)
-  def get_size(%{"unit" => unit, "value" => value}) when unit in ~w(y yb yottabyte yottabytes), do: parse_integer(value, @yb_10)
+  def as_size(%{"unit" => "", "value" => value}), do: parse_integer(value, 1)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(b byte bytes), do: parse_integer(value, 1)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(k kb kilobyte kilobytes), do: parse_integer(value, @kb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(m mb megabyte megabytes), do: parse_integer(value, @mb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(g gb gigabyte gigabytes), do: parse_integer(value, @gb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(t tb terabyte terabytes), do: parse_integer(value, @tb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(p pb petabyte petabytes), do: parse_integer(value, @pb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(e eb exabyte exabytes), do: parse_integer(value, @eb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(z zb zettabyte zettabytes), do: parse_integer(value, @zb_10)
+  def as_size(%{"unit" => unit, "value" => value}) when unit in ~w(y yb yottabyte yottabytes), do: parse_integer(value, @yb_10)
 
   defp parse_integer(string, factor) do
     with {result, ""} <- Integer.parse(string) do

lib/hocon/parser.ex

@@ -15,16 +15,29 @@ defmodule Hocon.Parser do
 
   """
   def decode(string, opts \\ []) do
+    try do
+      {:ok, decode!(string, opts)}
+    catch
+      error -> error
+    end
+
+  end
+
+  @doc"""
+  Similar to `decode/2` except it will unwrap the error tuple and raise
+  in case of errors.
+  """
+  def decode!(string, opts \\ []) do
     with {:ok, ast} <- Tokenizer.decode(string) do
       with {[], result } <- ast
                             |> contact_rule([])
-                            |> parse_root(),
-                result   <- Document.convert(result, opts) do
-        {:ok, result}
+                            |> parse_root() do
+        Document.convert(result, opts)
       end
     end
   end
 
+
   def contact_rule([], result) do
     Enum.reverse(result)
   end
@@ -137,7 +150,7 @@ defmodule Hocon.Parser do
     {rest, doc} = Document.put(result, to_string(key), value, rest)
     parse_object(rest, doc, root)
   end
-  defp parse_object(tokens, result, root) do
+  defp parse_object(_tokens, _result, _root) do
     throw {:error, "syntax error"}
   end
 

mix.exs

@@ -1,7 +1,7 @@
 defmodule Hocon.MixProject do
   use Mix.Project
 
-  @version "0.1.2"
+  @version "0.1.3"
 
   def project do
     [
@@ -31,8 +31,6 @@ defmodule Hocon.MixProject do
     [
       {:excoveralls, "~> 0.12.1", only: :test},
       {:ex_doc, "~> 0.21", only: :dev, runtime: false}
-      # {:dep_from_hexpm, "~> 0.3.0"},
-      # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}
     ]
   end
 

test/hocon_test.exs

@@ -5,12 +5,8 @@ defmodule HoconTest do
   alias Hocon.Tokenizer
 
   test "Parse and show the config.conf" do
-
     {:ok, body} = File.read("./test/data/config.conf")
-    result = Parser.decode(body)
-
-    #IO.puts inspect result
-
+    {:ok, _} = Parser.decode(body)
     assert true
   end
 
@@ -95,46 +91,13 @@ defmodule HoconTest do
     assert {:ok, %{"key" => "horse is my favorite animal"}} == Hocon.decode(~s(key : "horse " is my favorite animal))
   end
 
-  test "Parsing substitutions" do
-    assert {:ok, %{"key" => "${animal.favorite} is my favorite animal", "animal" => %{"favorite" => "dog"}}} == Hocon.decode(~s(animal { favorite : "dog" }, key : """${animal.favorite} is my favorite animal"""))
-    assert {:ok, %{"key" => "dog is my favorite animal", "animal" => %{"favorite" => "dog"}}} == Hocon.decode(~s(animal { favorite : "dog" }, key : ${animal.favorite} is my favorite animal))
-    assert {:ok, %{"key" => "dog is my favorite animal", "animal" => %{"favorite" => "dog"}}} == Hocon.decode(~s(animal { favorite : "dog" }, key : ${animal.favorite}" is my favorite animal"))
-    assert catch_throw(Hocon.decode(~s(key : ${animal.favorite}" is my favorite animal"))) == {:not_found, "animal.favorite"}
-    assert {:ok, %{"key" => "Max limit is 10", "limit" => %{"max" => 10}}} == Hocon.decode(~s(limit { max : 10 }, key : Max limit is ${limit.max}))
-    assert {:ok, %{"key" => "Max limit is ${limit.max}", "limit" => %{"max" => 10}}} == Hocon.decode(~s(limit { max : 10 }, key : """Max limit is ${limit.max}"""))
-    assert {:ok, %{"key" => "Max limit is ${limit.max}", "limit" => %{"max" => 10}}} == Hocon.decode(~s(limit { max : 10 }, key : "Max limit is ${limit.max}"))
-  end
-
-  test "Parsing complex substitutions" do
-    assert {:ok, %{"animal" => %{"favorite" => "dog"}, "a" => %{"b" => %{"c" => "dog"}}}} == Hocon.decode(~s(animal { favorite : "dog" }, a { b { c : ${animal.favorite}}}))
-    assert {:ok, %{"bar" => %{"baz" => 42, "foo" => 42}}} == Hocon.decode(~s(bar : { foo : 42, baz : ${bar.foo}}))
-    assert {:ok, %{"bar" => %{"baz" => 43, "foo" => 43}}} == Hocon.decode(~s(bar : { foo : 42, baz : ${bar.foo} }\nbar : { foo : 43 }))
-    assert {:ok, %{"bar" => %{"a" => 4, "b" => 3}, "foo" => %{"c" => 3, "d" => 4}}} == Hocon.decode(~s(bar : { a : ${foo.d}, b : 1 }\nbar.b = 3\nfoo : { c : ${bar.b}, d : 2 }\nfoo.d = 4))
-    assert {:ok, %{"a" => "2 2", "b" => 2}} == Hocon.decode(~s(a : ${b}, b : 2\n a : ${a} ${b}))
-  end
-
-  test "Parsing self-references substitutions" do
-    assert {:ok, %{"foo" => %{"a" => 2, "c" => 1}}} == Hocon.decode(~s(foo : { a : { c : 1 } }\nfoo : ${foo.a}\nfoo : { a : 2 }))
-    assert {:ok, %{"foo" => "1 2"}} == Hocon.decode(~s(foo : { bar : 1, baz : 2 }\nfoo : ${foo.bar} ${foo.baz}))
-    assert {:ok, %{"foo" => "1 2", "baz" => 2}} == Hocon.decode(~s(baz : 2\nfoo : { bar : 1, baz : 2 }\nfoo : ${foo.bar} ${baz}))
-    assert {:ok, %{"path" => "a:b:c:d"}} == Hocon.decode(~s(path : "a:b:c"\npath : ${path}":d"))
-    assert catch_throw(Hocon.decode(~s(foo : { bar : 1 }\nfoo : ${foo.bar} ${foo.baz}))) == {:not_found, "foo.foo.baz"}
-  end
 
   test "Parsing json" do
     assert {:ok, %{"a" => %{"b" => "c"}}} == Hocon.decode(~s({"a" : { "b" : "c"}}))
     assert {:ok, %{"a" => [1, 2, 3, 4]}} == Hocon.decode(~s({"a" : [1,2,3,4]}))
     assert {:ok, %{"a" => "b", "c" => ["a", "b", "c"], "x" => 10.99}} == Hocon.decode(~s({"a" : "b", "c" : ["a", "b", "c"], "x" : 10.99}))
   end
 
-  test "Parsing substitutions with cycles" do
-    assert catch_throw(Hocon.decode(~s(bar : ${foo}\nfoo : ${bar}))) == {:circle_detected, "foo"}
-    assert catch_throw(Hocon.decode(~s(a : ${b}\nb : ${c}\nc : ${a}))) == {:circle_detected, "b"}
-    assert catch_throw(Hocon.decode(~s(a : 1\nb : 2\na : ${b}\nb : ${a}))) == {:circle_detected, "b"}
-    assert catch_throw(Hocon.decode(~s(a : { b : ${a} }))) == {:circle_detected, "a"}
-    assert catch_throw(Hocon.decode(~s(a : { b : ${x} }))) == {:not_found, "x"}
-  end
-
   test "Parsing unquoted strings as values" do
     assert {:ok, %{"a" => "c"}} == Hocon.decode(~s({a : b\n a : c}))
   end
@@ -143,24 +106,4 @@ defmodule HoconTest do
     assert {:ok, %{"a" => %{"b" => %{"c" => 1}}}} == Hocon.decode(~s({"a" { "b" { c : 1 }}}))
   end
 
-  test "Parsing substitutions with environment variables" do
-    System.put_env("MY_HOME", "/home/greta")
-    assert {:ok, %{"path" => "/home/greta"}} == Hocon.decode(~s(path : ${MY_HOME}))
-    System.put_env("MY_HOME", "/home")
-    assert {:ok, %{"path" => "/home/greta"}} == Hocon.decode(~s(path : ${MY_HOME}\n path : ${path}"/greta"))
-    assert {:ok, %{"path" => ["/home", "usr/bin"]}} == Hocon.decode(~s(path : [${MY_HOME}]\n path : ${path} [ /usr/bin ]))
-  end
-
-  test "Parsing  += field separator" do
-    assert {:ok, %{"a" => [1, "a"]}} == Hocon.decode(~s(a += 1\n a+= a))
-    assert {:ok, %{"a" => [1, "a", 2, 3], "b" => 3}} == Hocon.decode(~s(b : 3, a += 1\n a+= a\n a += 2\n a += ${b}))
-    assert {:ok, %{"b" => 3, "dic" => %{"a" => [1, "a", 2, 3]}}} == Hocon.decode(~s(b : 3, dic { a += 1\n a+= a\n a += 2\n a += ${b} }))
-  end
-
-  test "Parsing optional substitutions " do
-    assert {:ok, %{"path" => ""}} == Hocon.decode(~s(path : ${?THE_HOME}))
-    assert {:ok, %{"a" => [1, "a", 2, ""]}} == Hocon.decode(~s(a += 1\n a+= a\n a += 2\n a += ${?b}))
-    assert {:ok, %{"bar" => %{"baz" => "", "fooz" => 42}}} == Hocon.decode(~s(bar : { fooz : 42, baz : ${?bar.foo}}))
-  end
-
 end

test/parser/basic_usage_test.exs

@@ -4,7 +4,8 @@ defmodule Parser.BasicUsageTest do
   test "parsing a simple object" do
     assert {:ok, %{"key" => "value"}} == Hocon.decode(~s(key = value))
     assert %{"key" => "value"} == Hocon.decode!(~s(key = value))
-    assert catch_throw(Hocon.decode(~s({a : b :}))) ==  {:error, "syntax error"}
+    assert {:error, "syntax error"} == Hocon.decode(~s({a : b :}))
+    assert catch_throw(Hocon.decode!(~s({a : b :}))) ==  {:error, "syntax error"}
     assert catch_throw(Hocon.decode!(~s({a : b :}))) ==  {:error, "syntax error"}
   end