Improve Updaters

.devcontainer/docker-compose.yml

@@ -31,6 +31,7 @@ services:
       POSTGRES_PASSWORD: health_board
     volumes:
       - postgres:/var/lib/postgresql/data
+      - ../.misc/data_updates:/app/.misc/data_updates
       - ../.misc/data:/app/.misc/data
     ports:
       - 5432:5432

.misc/data_updates/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.misc/docker/prod/build/Dockerfile

@@ -3,6 +3,8 @@ FROM erlang:22.3.4-alpine
 
 WORKDIR /app
 
+RUN apk add coreutils
+
 COPY --from=release /app/_build/prod/rel/health_board .
 COPY start.sh start.sh
 

config/.gitignore

@@ -0,0 +1 @@
+override.exs

config/config.exs

@@ -8,3 +8,9 @@ end
 
 wildcard_import.("general/*.exs")
 wildcard_import.("#{Mix.env()}/*.exs")
+
+override_file_path = Path.expand("override.exs", __DIR__)
+
+if File.exists?(override_file_path) do
+  import_config override_file_path
+end

config/general/health_board.exs

@@ -3,10 +3,12 @@ import Config
 config :health_board,
   basic_auth_dashboard_password: System.get_env("HEALTH_BOARD__BASIC_AUTH_DASHBOARD_PASSWORD", "Pass@123"),
   basic_auth_system_password: System.get_env("HEALTH_BOARD__BASIC_AUTH_SYSTEM_PASSWORD", "Pass@123"),
-  google_api_key: System.get_env("HEALTH_BOARD__GOOGLE_API_KEY"),
+  covid_reports_update_at_hour: 3,
   data_path: Path.join(File.cwd!(), ".misc/data"),
-  spreadsheet_page: System.get_env("HEALTH_BOARD__SPREADSHEET_PAGE"),
+  data_updates_path: Path.join(File.cwd!(), ".misc/data_updates"),
+  google_api_key: System.get_env("HEALTH_BOARD__GOOGLE_API_KEY"),
+  sars_update_at_hour: 3,
   split_command: System.get_env("HEALTH_BOARD__SPLIT_COMMAND", "split"),
   spreadsheet_id: System.get_env("HEALTH_BOARD__SPREADSHEET_ID"),
-  start_data_puller: false,
-  time_zone: -3
+  spreadsheet_page: System.get_env("HEALTH_BOARD__SPREADSHEET_PAGE"),
+  start_updater: String.to_existing_atom(System.get_env("HEALTH_BOARD__START_UPDATER", "false"))

config/general/logger.exs

@@ -5,4 +5,4 @@ config :logger,
 
 config :logger, :console,
   format: "$time $metadata[$level$levelpad] $message\n",
-  metadata: [:request_id]
+  metadata: [:mfa, :request_id]

config/releases.exs

@@ -41,7 +41,16 @@ defmodule HealthBoard.Releases.Helper do
 
   def health_board_settings do
     [
-      data_path: get_env!("DATA_PATH")
+      basic_auth_dashboard_password: get_env!("BASIC_AUTH_DASHBOARD_PASSWORD"),
+      basic_auth_system_password: get_env!("BASIC_AUTH_SYSTEM_PASSWORD"),
+      covid_reports_update_at_hour: get_env("COVID_REPORTS_UPDATE_AT_HOUR", :integer, 3),
+      data_path: get_env!("DATA_PATH"),
+      data_updates_path: get_env!("DATA_UPDATES_PATH"),
+      google_api_key: get_env!("GOOGLE_API_KEY"),
+      sars_update_at_hour: get_env("SARS_UPDATE_AT_HOUR", :integer, 3),
+      spreadsheet_id: get_env!("SPREADSHEET_ID"),
+      spreadsheet_page: get_env!("SPREADSHEET_PAGE"),
+      start_updater: get_env("START_UPDATER", :boolean, true)
     ]
   end
 

lib/health_board/application.ex

@@ -1,26 +1,22 @@
 defmodule HealthBoard.Application do
-  alias HealthBoard.Release.DataPuller.DataPullerSupervisor
   use Application
-
-  @app :health_board
+  alias HealthBoard.Updaters
 
   @spec start(any(), any()) :: {:ok, pid} | {:error, any()}
   def start(_type, _args) do
-    children = get_children()
-
     opts = [strategy: :one_for_one, name: HealthBoard.Supervisor]
-    Supervisor.start_link(children, opts)
+    Supervisor.start_link(children(), opts)
   end
 
-  defp get_children do
+  defp children do
     children = [
       HealthBoard.Repo,
       {Phoenix.PubSub, name: HealthBoard.PubSub},
       HealthBoardWeb.Endpoint
     ]
 
-    if Application.fetch_env!(@app, :start_data_puller) do
-      children ++ [DataPullerSupervisor]
+    if Application.fetch_env!(:health_board, :start_updater) do
+      children ++ [Updaters.Supervisor]
     else
       children
     end

lib/health_board/contexts/managers/icu_occupancy/daily_icu_occupancy.ex → lib/health_board/contexts/managers/hospital_capacity/daily_icu_rate.ex

@@ -1,11 +1,11 @@
-defmodule HealthBoard.Contexts.ICUOccupancy.DailyICUOccupancy do
+defmodule HealthBoard.Contexts.HospitalCapacity.DailyICURate do
   import Ecto.Query, only: [order_by: 2, where: 2, dynamic: 1, dynamic: 2]
-  alias HealthBoard.Contexts.ICUOccupancy.DayICUOccupancy
+  alias HealthBoard.Contexts.HospitalCapacity.DayICURate
   alias HealthBoard.Repo
 
-  @type schema :: %DayICUOccupancy{}
+  @type schema :: %DayICURate{}
 
-  @schema DayICUOccupancy
+  @schema DayICURate
 
   @spec new(keyword) :: schema
   def new(params \\ []) do

lib/health_board/contexts/managers/info/sources.ex

@@ -0,0 +1,25 @@
+defmodule HealthBoard.Contexts.Info.Sources do
+  alias HealthBoard.Contexts.Info.Source
+  alias HealthBoard.Repo
+
+  @type schema :: %Source{}
+
+  @schema Source
+
+  @spec get(String.t()) :: {:ok, schema} | {:error, :not_found}
+  def get(id) do
+    case Repo.get(@schema, id) do
+      nil -> {:error, :not_found}
+      struct -> {:ok, struct}
+    end
+  end
+
+  @spec update(String.t(), map) :: {:ok, schema} | {:error, Ecto.Changeset.t()}
+  def update(id, params) do
+    with {:ok, struct} <- get(id) do
+      struct
+      |> @schema.changeset(params)
+      |> Repo.update()
+    end
+  end
+end

lib/health_board/contexts/schemas/icu_occupancy/day_icu_occupancy.ex → lib/health_board/contexts/schemas/hospital_capacity/day_icu_rate.ex

@@ -1,8 +1,8 @@
-defmodule HealthBoard.Contexts.ICUOccupancy.DayICUOccupancy do
+defmodule HealthBoard.Contexts.HospitalCapacity.DayICURate do
   use Ecto.Schema
   alias HealthBoard.Contexts.Geo
 
-  schema "daily_covid_reports" do
+  schema "daily_icu_rate" do
     field :date, :date, null: false
 
     field :rate, :integer, default: 0

lib/health_board/contexts/schemas/info/source.ex

@@ -1,5 +1,6 @@
 defmodule HealthBoard.Contexts.Info.Source do
   use Ecto.Schema
+  import Ecto.Changeset, only: [cast: 3]
 
   @type schema :: %__MODULE__{}
 
@@ -15,4 +16,9 @@ defmodule HealthBoard.Contexts.Info.Source do
     field :last_update_date, :date
     field :extraction_date, :date
   end
+
+  @spec changeset(struct, map) :: Ecto.Changeset.t()
+  def changeset(struct, params \\ %{}) do
+    cast(struct, params, [:last_update_date, :extraction_date])
+  end
 end

lib/health_board/contexts/seeder.ex

@@ -0,0 +1,57 @@
+defmodule HealthBoard.Contexts.Seeder do
+  alias HealthBoard.Repo
+  require Logger
+
+  @spec csv_from_context!(String.t(), String.t(), list(atom), String.t() | nil) :: :ok
+  def csv_from_context!(context, table_name, fields, base_path \\ nil) do
+    base_path
+    |> Kernel.||(data_path())
+    |> Path.join(context)
+    |> Path.join(table_name <> ".csv")
+    |> csv_from_file_path!(table_name, fields)
+  end
+
+  @spec csvs_from_context!(String.t(), String.t(), list(atom), String.t() | nil) :: :ok
+  def csvs_from_context!(context, table_name, fields, base_path \\ nil) do
+    context
+    |> Path.join(table_name)
+    |> csvs_from_path!(table_name, fields, base_path)
+  end
+
+  @spec csvs_from_path!(String.t(), String.t(), list(atom), String.t() | nil) :: :ok
+  def csvs_from_path!(path, table_name, fields, base_path \\ nil) do
+    path = Path.join(base_path || data_path(), path)
+
+    path
+    |> File.ls!()
+    |> Enum.sort()
+    |> Enum.map(&Path.join(path, &1))
+    |> Enum.each(&csv_from_file_path!(&1, table_name, fields))
+  end
+
+  @spec csv_from_file_path!(String.t(), String.t(), list(atom)) :: :ok
+  def csv_from_file_path!(path, table_name, fields) do
+    Logger.info(~s(Seeding #{Path.basename(path)} for table #{table_name}))
+
+    path
+    |> csv_copy_query(table_name, fields)
+    |> Repo.query!()
+
+    :ok
+  end
+
+  defp csv_copy_query(csv_path, table_name, fields) do
+    fields = Enum.join(fields, ",")
+    "COPY #{table_name}(#{fields}) FROM '#{csv_path}' WITH CSV;"
+  end
+
+  defp data_path do
+    Application.get_env(:health_board, :data_path)
+  end
+
+  @spec down!(String.t()) :: :ok
+  def down!(table_name) do
+    Repo.query!("TRUNCATE #{table_name} CASCADE;")
+    :ok
+  end
+end

lib/health_board/contexts/seeders/demographic.ex

@@ -0,0 +1,28 @@
+defmodule HealthBoard.Contexts.Seeders.Demographic do
+  alias HealthBoard.Contexts.Seeders
+
+  @spec down!(keyword) :: :ok
+  def down!(opts \\ []) do
+    what = Keyword.get(opts, :what, :all)
+
+    if what in [:all, :yearly_populations], do: Seeders.YearlyPopulations.down!()
+
+    :ok
+  end
+
+  @spec reseed!(keyword) :: :ok
+  def reseed!(opts \\ []) do
+    down!(opts)
+    up!(opts)
+  end
+
+  @spec up!(keyword) :: :ok
+  def up!(opts \\ []) do
+    what = Keyword.get(opts, :what, :all)
+    base_path = Keyword.get(opts, :base_path)
+
+    if what in [:all, :yearly_populations], do: Seeders.YearlyPopulations.up!(base_path)
+
+    :ok
+  end
+end

lib/health_board/release/data_manager/demographic/yearly_populations.ex → lib/health_board/contexts/seeders/demographic/yearly_populations.ex

@@ -1,6 +1,5 @@
-defmodule HealthBoard.Release.DataManager.YearlyPopulations do
-  alias HealthBoard.Release.DataManager
-  alias HealthBoard.Repo
+defmodule HealthBoard.Contexts.Seeders.YearlyPopulations do
+  alias HealthBoard.Contexts.Seeder
 
   @context "demographic"
   @table_name "yearly_populations"
@@ -28,14 +27,15 @@ defmodule HealthBoard.Release.DataManager.YearlyPopulations do
     :age_80_or_more
   ]
 
-  @spec up :: :ok
-  def up do
-    DataManager.copy!(@context, @table_name, @columns)
-  end
+  @spec down! :: :ok
+  def down!, do: Seeder.down!(@table_name)
 
-  @spec down :: :ok
-  def down do
-    Repo.query!("TRUNCATE #{@table_name} CASCADE;")
-    :ok
+  @spec reseed!(String.t() | nil) :: :ok
+  def reseed!(base_path \\ nil) do
+    up!(base_path)
+    down!()
   end
+
+  @spec up!(String.t() | nil) :: :ok
+  def up!(base_path \\ nil), do: Seeder.csv_from_context!(@context, @table_name, @columns, base_path)
 end

lib/health_board/contexts/seeders/flu_syndrome.ex

@@ -0,0 +1,34 @@
+defmodule HealthBoard.Contexts.Seeders.FluSyndrome do
+  alias HealthBoard.Contexts.Seeders
+
+  @spec down!(keyword) :: :ok
+  def down!(opts \\ []) do
+    what = Keyword.get(opts, :what, :all)
+
+    if what in [:all, :flu_syndrome, :daily_flu_syndrome_cases], do: Seeders.DailyFluSyndromeCases.down!()
+    if what in [:all, :flu_syndrome, :weekly_flu_syndrome_cases], do: Seeders.WeeklyFluSyndromeCases.down!()
+    if what in [:all, :flu_syndrome, :monthly_flu_syndrome_cases], do: Seeders.MonthlyFluSyndromeCases.down!()
+    if what in [:all, :flu_syndrome, :pandemic_flu_syndrome_cases], do: Seeders.PandemicFluSyndromeCases.down!()
+
+    :ok
+  end
+
+  @spec reseed!(keyword) :: :ok
+  def reseed!(opts \\ []) do
+    down!(opts)
+    up!(opts)
+  end
+
+  @spec up!(keyword) :: :ok
+  def up!(opts \\ []) do
+    what = Keyword.get(opts, :what, :all)
+    base_path = Keyword.get(opts, :base_path)
+
+    if what in [:all, :flu_syndrome, :pandemic_flu_syndrome_cases], do: Seeders.PandemicFluSyndromeCases.up!(base_path)
+    if what in [:all, :flu_syndrome, :monthly_flu_syndrome_cases], do: Seeders.MonthlyFluSyndromeCases.up!(base_path)
+    if what in [:all, :flu_syndrome, :weekly_flu_syndrome_cases], do: Seeders.WeeklyFluSyndromeCases.up!(base_path)
+    if what in [:all, :flu_syndrome, :daily_flu_syndrome_cases], do: Seeders.DailyFluSyndromeCases.up!(base_path)
+
+    :ok
+  end
+end

lib/health_board/release/data_manager/flu_syndrome/daily_flu_syndrome_cases.ex → lib/health_board/contexts/seeders/flu_syndrome/daily_flu_syndrome_cases.ex

@@ -1,6 +1,5 @@
-defmodule HealthBoard.Release.DataManager.DailyFluSyndromeCases do
-  alias HealthBoard.Release.DataManager
-  alias HealthBoard.Repo
+defmodule HealthBoard.Contexts.Seeders.DailyFluSyndromeCases do
+  alias HealthBoard.Contexts.Seeder
 
   @context "flu_syndrome"
   @table_name "daily_flu_syndrome_cases"
@@ -47,16 +46,15 @@ defmodule HealthBoard.Release.DataManager.DailyFluSyndromeCases do
     :health_professional
   ]
 
-  @spec up :: :ok
-  def up do
-    @context
-    |> Path.join(@table_name)
-    |> DataManager.copy_from_dir!(@table_name, @columns)
-  end
+  @spec down! :: :ok
+  def down!, do: Seeder.down!(@table_name)
 
-  @spec down :: :ok
-  def down do
-    Repo.query!("TRUNCATE #{@table_name} CASCADE;")
-    :ok
+  @spec reseed!(String.t() | nil) :: :ok
+  def reseed!(base_path \\ nil) do
+    up!(base_path)
+    down!()
   end
+
+  @spec up!(String.t() | nil) :: :ok
+  def up!(base_path \\ nil), do: Seeder.csvs_from_context!(@context, @table_name, @columns, base_path)
 end

lib/health_board/release/data_manager/flu_syndrome/monthly_flu_syndrome_cases.ex → lib/health_board/contexts/seeders/flu_syndrome/monthly_flu_syndrome_cases.ex

@@ -1,6 +1,5 @@
-defmodule HealthBoard.Release.DataManager.MonthlyFluSyndromeCases do
-  alias HealthBoard.Release.DataManager
-  alias HealthBoard.Repo
+defmodule HealthBoard.Contexts.Seeders.MonthlyFluSyndromeCases do
+  alias HealthBoard.Contexts.Seeder
 
   @context "flu_syndrome"
   @table_name "monthly_flu_syndrome_cases"
@@ -48,14 +47,15 @@ defmodule HealthBoard.Release.DataManager.MonthlyFluSyndromeCases do
     :health_professional
   ]
 
-  @spec up :: :ok
-  def up do
-    DataManager.copy!(@context, @table_name, @columns)
-  end
+  @spec down! :: :ok
+  def down!, do: Seeder.down!(@table_name)
 
-  @spec down :: :ok
-  def down do
-    Repo.query!("TRUNCATE #{@table_name} CASCADE;")
-    :ok
+  @spec reseed!(String.t() | nil) :: :ok
+  def reseed!(base_path \\ nil) do
+    up!(base_path)
+    down!()
   end
+
+  @spec up!(String.t() | nil) :: :ok
+  def up!(base_path \\ nil), do: Seeder.csv_from_context!(@context, @table_name, @columns, base_path)
 end