encoding and stuff

bench/encode.exs

@@ -20,21 +20,21 @@ types = [:u32, :string, :datetime, {:array, :string}]
 
 Benchee.run(
   %{
-    "csv stream" => fn ->
-      rows |> NimbleCSV.RFC4180.dump_to_stream() |> Stream.run()
-    end,
-    "csv" => fn ->
-      NimbleCSV.RFC4180.dump_to_iodata(rows)
-    end,
-    "encode_rows + csv stream" => fn ->
-      rows |> Stream.map(&CSV.encode_row/1) |> NimbleCSV.RFC4180.dump_to_stream() |> Stream.run()
-    end,
-    "encode_rows + csv" => fn ->
-      rows |> CSV.encode_rows() |> NimbleCSV.RFC4180.dump_to_iodata()
-    end,
-    "row_binary stream" => fn ->
-      rows |> Stream.map(&RowBinary.encode_row(&1, types)) |> Stream.run()
-    end,
+    # "csv stream" => fn ->
+    #   rows |> NimbleCSV.RFC4180.dump_to_stream() |> Stream.run()
+    # end,
+    # "csv" => fn ->
+    #   NimbleCSV.RFC4180.dump_to_iodata(rows)
+    # end,
+    # "encode_rows + csv stream" => fn ->
+    #   rows |> Stream.map(&CSV.encode_row/1) |> NimbleCSV.RFC4180.dump_to_stream() |> Stream.run()
+    # end,
+    # "encode_rows + csv" => fn ->
+    #   rows |> CSV.encode_rows() |> NimbleCSV.RFC4180.dump_to_iodata()
+    # end,
+    # "row_binary stream" => fn ->
+    #   rows |> Stream.map(&RowBinary.encode_row(&1, types)) |> Stream.run()
+    # end,
     "row_binary" => fn ->
       RowBinary.encode_rows(rows, types)
     end

lib/ch/row_binary.ex

@@ -18,25 +18,29 @@ defmodule Ch.RowBinary do
   def encode_row([] = done, []), do: done
 
   def encode_rows([row | rows], types), do: encode_rows(row, types, rows, types)
-  def encode_rows([] = done, _types), do: done
+  def encode_rows([] = empty, _types), do: empty
 
   defp encode_rows([el | els], [t | ts], rows, types) do
     [encode(t, el) | encode_rows(els, ts, rows, types)]
   end
 
-  defp encode_rows([], [], rows, types), do: encode_rows(rows, types)
+  defp encode_rows([], [], [row | rows], types) do
+    encode_rows(row, types, rows, types)
+  end
 
-  def encode(:varint, num) when is_integer(num) and num < 128, do: <<num>>
+  defp encode_rows([], [], [] = done, _types), do: done
 
-  def encode(:varint, num) when is_integer(num) do
-    [<<1::1, num::7>> | encode(:varint, num >>> 7)]
-  end
+  @doc false
+  def encode_varint(num) when num <= 0x7F, do: num
+  def encode_varint(num), do: [num &&& 0x7F | encode_varint_cont(num >>> 7)]
+  defp encode_varint_cont(num) when num <= 0x7F, do: [num]
+  defp encode_varint_cont(num), do: [num &&& 0x7F | encode_varint_cont(num >>> 7)]
 
   def encode(type, str) when type in [:string, :binary] do
     case str do
-      _ when is_binary(str) -> [encode(:varint, byte_size(str)) | str]
-      _ when is_list(str) -> [encode(:varint, IO.iodata_length(str)) | str]
-      nil -> <<0>>
+      _ when is_binary(str) -> [encode_varint(byte_size(str)) | str]
+      _ when is_list(str) -> [encode_varint(IO.iodata_length(str)) | str]
+      nil -> 0
     end
   end
 
@@ -51,7 +55,7 @@ defmodule Ch.RowBinary do
 
   def encode(string(size: size), nil), do: <<0::size(size * 8)>>
 
-  for size <- [8, 16, 32, 64, 128, 256] do
+  for size <- [32, 64, 8, 16, 128, 256] do
     def encode(unquote(:"u#{size}"), i) when is_integer(i) do
       <<i::unquote(size)-little>>
     end
@@ -72,34 +76,27 @@ defmodule Ch.RowBinary do
     def encode(unquote(:"f#{size}"), nil), do: <<0::unquote(size)>>
   end
 
-  def encode(decimal(size: size, scale: scale), %Decimal{sign: sign, coef: coef, exp: exp})
-      when scale == -exp do
-    i = sign * coef
-    <<i::size(size)-little>>
-  end
-
-  def encode(decimal(size: size, scale: scale), %Decimal{sign: sign, coef: coef, exp: exp})
-      when exp >= 0 do
-    i = sign * coef * round(:math.pow(10, exp + scale))
-    <<i::size(size)-little>>
+  def encode(:boolean, bool) do
+    case bool do
+      true -> 1
+      false -> 0
+      nil -> 0
+    end
   end
 
-  def encode(decimal(scale: scale) = t, %Decimal{} = d) do
-    encode(t, Decimal.round(d, scale))
+  def encode({:array, type}, l) do
+    case l do
+      [_ | _] -> [encode_varint(length(l)) | encode_many(type, l)]
+      [] -> 0
+      nil -> 0
+    end
   end
 
-  def encode(decimal(size: size), nil), do: <<0::size(size)>>
-
-  def encode(:boolean, true), do: <<1>>
-  def encode(:boolean, false), do: <<0>>
-  def encode(:boolean, nil), do: <<0>>
-
-  def encode({:array, type}, [_ | _] = l) do
-    [encode(:varint, length(l)) | encode_many(l, type)]
+  def encode(:date, %Date{} = date) do
+    <<Date.diff(date, @epoch_date)::16-little>>
   end
 
-  def encode({:array, _type}, []), do: <<0>>
-  def encode({:array, _type}, nil), do: <<0>>
+  def encode(:date, nil), do: <<0::16>>
 
   def encode(:datetime, %NaiveDateTime{} = datetime) do
     <<NaiveDateTime.diff(datetime, @epoch_naive_datetime)::32-little>>
@@ -117,6 +114,24 @@ defmodule Ch.RowBinary do
     encode(t, v)
   end
 
+  def encode(decimal(size: size, scale: scale), %Decimal{sign: sign, coef: coef, exp: exp})
+      when scale == -exp do
+    i = sign * coef
+    <<i::size(size)-little>>
+  end
+
+  def encode(decimal(size: size, scale: scale), %Decimal{sign: sign, coef: coef, exp: exp})
+      when exp >= 0 do
+    i = sign * coef * round(:math.pow(10, exp + scale))
+    <<i::size(size)-little>>
+  end
+
+  def encode(decimal(scale: scale) = t, %Decimal{} = d) do
+    encode(t, Decimal.round(d, scale))
+  end
+
+  def encode(decimal(size: size), nil), do: <<0::size(size)>>
+
   def encode(datetime64(unit: unit), %NaiveDateTime{} = datetime) do
     <<NaiveDateTime.diff(datetime, @epoch_naive_datetime, unit)::64-little-signed>>
   end
@@ -127,12 +142,6 @@ defmodule Ch.RowBinary do
 
   def encode(datetime64(), nil), do: <<0::64>>
 
-  def encode(:date, %Date{} = date) do
-    <<Date.diff(date, @epoch_date)::16-little>>
-  end
-
-  def encode(:date, nil), do: <<0::16>>
-
   def encode(:date32, %Date{} = date) do
     <<Date.diff(date, @epoch_date)::32-little-signed>>
   end
@@ -158,10 +167,19 @@ defmodule Ch.RowBinary do
   def encode(:uuid, nil), do: <<0::128>>
 
   def encode({:nullable, _type}, nil), do: 1
-  def encode({:nullable, type}, value), do: [0 | encode(type, value)]
+  def encode({:nullable, type}, value), do: [0 | ensure_iodata(encode(type, value))]
+
+  defp encode_many(type, values) do
+    case values do
+      [el | rest] -> [encode(type, el) | encode_many(type, rest)]
+      [] = done -> done
+    end
+  end
 
-  defp encode_many([el | rest], type), do: [encode(type, el) | encode_many(rest, type)]
-  defp encode_many([] = done, _type), do: done
+  @compile inline: [ensure_iodata: 1]
+  defp ensure_iodata(b) when is_binary(b), do: b
+  defp ensure_iodata(l) when is_list(l), do: l
+  defp ensure_iodata(i) when is_integer(i), do: [i]
 
   @compile {:inline, d: 1}
 
@@ -220,6 +238,7 @@ defmodule Ch.RowBinary do
 
   def encode_type({:nullable, type}), do: ["Nullable(", encode_type(type), ?)]
   def encode_type({:array, type}), do: ["Array(", encode_type(type), ?)]
+  def encode_type({:low_cardinality, type}), do: ["LowCardinality(", encode_type(type), ?)]
   def encode_type(:datetime), do: "DateTime"
 
   def encode_type({:datetime, timezone}) when is_binary(timezone) do

test/ch/row_binary_test.exs

@@ -85,6 +85,11 @@ defmodule Ch.RowBinaryTest do
       {{:nullable, :string}, "string"}
     ]
 
+    for {type, expected} <- spec do
+      encoded = IO.iodata_to_binary(encode(type, expected)) |> IO.inspect()
+      assert decode_rows(encoded, [type]) == [[expected]]
+    end
+
     num_cols = length(spec)
     {types, row} = Enum.unzip(spec)
 
@@ -123,7 +128,8 @@ defmodule Ch.RowBinaryTest do
 
     test "nil" do
       assert encode({:nullable, :string}, nil) == 1
-      assert encode(:string, nil) == <<0>>
+      assert encode({:nullable, :string}, "") == [0, 0 | ""]
+      assert encode(:string, nil) == 0
       assert encode(string(size: 2), nil) == <<0, 0>>
       assert encode(:u8, nil) == <<0>>
       assert encode(:u16, nil) == <<0, 0>>
@@ -135,8 +141,8 @@ defmodule Ch.RowBinaryTest do
       assert encode(:i64, nil) == <<0, 0, 0, 0, 0, 0, 0, 0>>
       assert encode(:f32, nil) == <<0, 0, 0, 0>>
       assert encode(:f64, nil) == <<0, 0, 0, 0, 0, 0, 0, 0>>
-      assert encode(:boolean, nil) == <<0>>
-      assert encode({:array, :string}, nil) == <<0>>
+      assert encode(:boolean, nil) == 0
+      assert encode({:array, :string}, nil) == 0
       assert encode(:date, nil) == <<0, 0>>
       assert encode(:date32, nil) == <<0, 0, 0, 0>>
       assert encode(:datetime, nil) == <<0, 0, 0, 0>>