Refactor smart_city to contain dataset and org update events

#39

co-authored-by: Cameron Marsh <cmarsh@pillartechnology.com>

.gitignore

@@ -22,4 +22,5 @@ erl_crash.dump
 # Ignore package tarball (built via "mix hex.build").
 scos_ex-*.tar
 
-.elixir_ls
+.elixir_ls
+.DS_Store

lib/smart_city/event/_metadata.ex

@@ -0,0 +1,60 @@
+defmodule SmartCity.Event.DatasetUpdate.Metadata do
+  @moduledoc """
+  A struct defining internal metadata on a registry event message.
+  """
+
+  alias SmartCity.Helpers
+
+  @type t :: %SmartCity.Event.DatasetUpdate.Metadata{
+          intendedUse: list(),
+          expectedBenefit: list()
+        }
+
+  @derive Jason.Encoder
+  defstruct intendedUse: [],
+            expectedBenefit: []
+
+  @doc """
+  Returns a new `SmartCity.Event.DatasetUpdate.Metadata` struct.
+  Can be created from `Map` with string or atom keys.
+  Raises an `ArgumentError` when passed invalid input.
+
+  ## Parameters
+
+    - msg: Map with string or atom keys that defines the dataset's metadata.
+
+  ## Examples
+
+      iex> SmartCity.Event.DatasetUpdate.Metadata.new(%{"intendedUse" => ["a","b","c"], "expectedBenefit" => [1,2,3]})
+      %SmartCity.Event.DatasetUpdate.Metadata{
+        expectedBenefit: [1, 2, 3],
+        intendedUse: ["a", "b", "c"]
+      }
+
+      iex> SmartCity.Event.DatasetUpdate.Metadata.new(%{:intendedUse => ["a","b","c"], :expectedBenefit => [1,2,3]})
+      %SmartCity.Event.DatasetUpdate.Metadata{
+        expectedBenefit: [1, 2, 3],
+        intendedUse: ["a", "b", "c"]
+      }
+
+      iex> SmartCity.Event.DatasetUpdate.Metadata.new("Not a map")
+      ** (ArgumentError) Invalid internal metadata: "Not a map"
+  """
+  @spec new(map()) :: SmartCity.Event.DatasetUpdate.Metadata.t()
+  def new(%{} = msg) do
+    msg_atoms =
+      case is_binary(List.first(Map.keys(msg))) do
+        true ->
+          Helpers.to_atom_keys(msg)
+
+        false ->
+          msg
+      end
+
+    struct(%__MODULE__{}, msg_atoms)
+  end
+
+  def new(msg) do
+    raise ArgumentError, "Invalid internal metadata: #{inspect(msg)}"
+  end
+end

lib/smart_city/event/business.ex

@@ -0,0 +1,103 @@
+defmodule SmartCity.Event.DatasetUpdate.Business do
+  @moduledoc """
+  A struct representing the business data portion of a dataset definition (represented by `SmartCity.Event.DatasetUpdate`)
+
+  You probably won't need to access this module directly; `SmartCity.Event.DatasetUpdate.new/1` will build this for you
+  """
+
+  alias SmartCity.Helpers
+
+  @type not_required :: term() | nil
+  @type license_or_default :: String.t()
+
+  @type t :: %SmartCity.Event.DatasetUpdate.Business{
+          dataTitle: String.t(),
+          description: String.t(),
+          modifiedDate: String.t(),
+          orgTitle: String.t(),
+          contactName: String.t(),
+          contactEmail: String.t(),
+          authorName: not_required(),
+          authorEmail: not_required(),
+          categories: not_required(),
+          conformsToUri: not_required(),
+          describedByMimeType: not_required(),
+          describedByUrl: not_required(),
+          homepage: not_required(),
+          issuedDate: not_required(),
+          keywords: not_required(),
+          language: not_required(),
+          license: license_or_default(),
+          parentDataset: not_required(),
+          publishFrequency: not_required(),
+          referenceUrls: not_required(),
+          rights: not_required(),
+          spatial: not_required(),
+          temporal: not_required()
+        }
+
+  @derive Jason.Encoder
+  defstruct dataTitle: nil,
+            description: nil,
+            modifiedDate: nil,
+            orgTitle: nil,
+            contactName: nil,
+            contactEmail: nil,
+            authorName: nil,
+            authorEmail: nil,
+            license: nil,
+            keywords: nil,
+            rights: nil,
+            homepage: nil,
+            spatial: nil,
+            temporal: nil,
+            publishFrequency: nil,
+            conformsToUri: nil,
+            describedByUrl: nil,
+            describedByMimeType: nil,
+            parentDataset: nil,
+            issuedDate: nil,
+            language: nil,
+            referenceUrls: nil,
+            categories: nil
+
+  @doc """
+  Returns a new `SmartCity.Event.DatasetUpdate.Business` struct.
+  Can be created from `Map` with string or atom keys.
+
+  ## Parameters
+    - msg: Map with string or atom keys that defines the dataset's business metadata. See `SmartCity.Event.DatasetUpdate.Business` typespec for available keys.
+
+    _Required Keys_
+      - dataTitle
+      - description
+      - modifiedDate
+      - orgTitle
+      - contactName
+      - contactEmail
+
+    * License will default to [http://opendefinition.org/licenses/cc-by/](http://opendefinition.org/licenses/cc-by/) if not provided
+  """
+  def new(%{"dataTitle" => _} = msg) do
+    msg
+    |> Helpers.to_atom_keys()
+    |> new()
+  end
+
+  def new(
+        %{
+          dataTitle: _,
+          description: _,
+          modifiedDate: _,
+          orgTitle: _,
+          contactName: _,
+          contactEmail: _
+        } = msg
+      ) do
+    struct(%__MODULE__{}, msg)
+  end
+
+  def new(msg) do
+    raise ArgumentError, "Invalid business metadata: #{inspect(msg)}"
+  end
+end

lib/smart_city/event/dataset_update.ex

@@ -0,0 +1,166 @@
+defmodule SmartCity.Event.DatasetUpdate do
+  @moduledoc """
+  Struct defining a dataset update event.  This is triggered when new datasets are put into the system or existing
+  datasets are updated
+
+  ```javascript
+  const Dataset = {
+    "id": "",                  // UUID
+    "business": {              // Project Open Data Metadata Schema v1.1
+      "dataTitle": "",       // user friendly (dataTitle)
+      "description": "",
+      "keywords": [""],
+      "modifiedDate": "",
+      "orgTitle": "",        // user friendly (orgTitle)
+      "contactName": "",
+      "contactEmail": "",
+      "license": "",
+      "rights": "",
+      "homepage": "",
+      "spatial": "",
+      "temporal": "",
+      "publishFrequency": "",
+      "conformsToUri": "",
+      "describedByUrl": "",
+      "describedByMimeType": "",
+      "parentDataset": "",
+      "issuedDate": "",
+      "language": "",
+      "referenceUrls": [""],
+      "categories": [""]
+    },
+    "technical": {
+      "dataName": "",        // ~r/[a-zA-Z_]+$/
+      "orgId": "",
+      "orgName": "",         // ~r/[a-zA-Z_]+$/
+      "systemName": "",      // ${orgName}__${dataName},
+      "schema": [
+        {
+          "name": "",
+          "type": "",
+          "description": ""
+        }
+      ],
+      "sourceUrl": "",
+      "protocol": "",       // List of protocols to use. Defaults to nil. Can be [http1, http2]
+      "authUrl": "",
+      "sourceFormat": "",
+      "sourceType": "",     // remote|stream|ingest|host
+      "cadence": "",
+      "sourceQueryParams": {
+        "key1": "",
+        "key2": ""
+      },
+      "transformations": [], // ?
+      "validations": [],     // ?
+      "sourceHeaders": {
+        "header1": "",
+        "header2": ""
+      }
+      "authHeaders": {
+        "header1": "",
+        "header2": ""
+      }
+    },
+    "_metadata": {
+      "intendedUse": [],
+      "expectedBenefit": []
+    }
+  }
+  ```
+  """
+
+  use SmartCity.Event.EventHelper
+  alias SmartCity.Event.DatasetUpdate.Business
+  alias SmartCity.Event.DatasetUpdate.Technical
+  alias SmartCity.Helpers
+  alias SmartCity.Event.DatasetUpdate.Metadata
+
+  @type id :: term()
+  @type t :: %SmartCity.Event.DatasetUpdate{
+          version: String.t(),
+          id: String.t(),
+          business: SmartCity.Event.DatasetUpdate.Business.t(),
+          technical: SmartCity.Event.DatasetUpdate.Technical.t(),
+          _metadata: SmartCity.Event.DatasetUpdate.Metadata.t()
+        }
+
+  @derive Jason.Encoder
+  defstruct version: "0.3", id: nil, business: nil, technical: nil, _metadata: nil
+
+  @doc """
+  Returns a new `SmartCity.Event.DatasetUpdate` struct. `SmartCity.Event.DatasetUpdate.Business`,
+  `SmartCity.Event.DatasetUpdate.Technical`, and `SmartCity.Event.DatasetUpdate.Metadata` structs will be created along the way.
+
+  ## Parameters
+
+  - msg : map defining values of the struct to be created.
+    Can be initialized by
+    - map with string keys
+    - map with atom keys
+    - JSON
+  """
+
+  def create(%{id: id, business: biz, technical: tech, _metadata: meta}) do
+    struct =
+      struct(%__MODULE__{}, %{
+        id: id,
+        business: Business.new(biz),
+        technical: Technical.new(tech),
+        _metadata: Metadata.new(meta)
+      })
+
+    {:ok, struct}
+  rescue
+    e -> {:error, e}
+  end
+
+  def create(%{id: id, business: biz, technical: tech}) do
+    create(%{id: id, business: biz, technical: tech, _metadata: %{}})
+  end
+
+  def create(msg) do
+    {:error, "Invalid registry message: #{inspect(msg)}"}
+  end
+
+  @doc """
+  Returns true if `SmartCity.Dataset.Technical sourceType field is stream`
+  """
+  def is_stream?(%__MODULE__{technical: %{sourceType: sourceType}}) do
+    "stream" == sourceType
+  end
+
+  @doc """
+  Returns true if `SmartCity.Dataset.Technical sourceType field is remote`
+  """
+  def is_remote?(%__MODULE__{technical: %{sourceType: sourceType}}) do
+    "remote" == sourceType
+  end
+
+  @doc """
+  Returns true if `SmartCity.Dataset.Technical sourceType field is ingest`
+  """
+  def is_ingest?(%__MODULE__{technical: %{sourceType: sourceType}}) do
+    "ingest" == sourceType
+  end
+
+  @doc """
+  Returns true if `SmartCity.Dataset.Technical sourceType field is host`
+  """
+  def is_host?(%__MODULE__{technical: %{sourceType: sourceType}}) do
+    "host" == sourceType
+  end
+
+  defp to_dataset(%{} = map) do
+    {:ok, dataset} = new(map)
+    dataset
+  end
+
+  defp to_dataset(json) do
+    json
+    |> Jason.decode!()
+    |> to_dataset()
+  end
+
+  defp ok(value), do: {:ok, value}
+end

lib/smart_city/event/event_helper.ex

@@ -0,0 +1,30 @@
+defmodule SmartCity.Event.EventHelper do
+  @moduledoc """
+  Macro for repeated code to deserialize and atomize event structs
+  """
+  @callback create(%{required(atom()) => term()}) :: term()
+  alias SmartCity.Helpers
+
+  defmacro __using__(_opts) do
+    quote do
+      @behaviour EventHelper
+
+      @spec new(String.t() | map()) :: {:ok, map()} | {:error, term()}
+      def new(msg) when is_binary(msg) do
+        with {:ok, decoded} <- Jason.decode(msg, keys: :atoms) do
+          create(decoded)
+        end
+      end
+
+      def new(%{"id" => _} = msg) do
+        msg
+        |> Helpers.to_atom_keys()
+        |> create()
+      end
+
+      def new(msg) do
+        create(msg)
+      end
+    end
+  end
+end