From 7da81a22a427d26102811d18b5424a67b522ce38 Mon Sep 17 00:00:00 2001 From: kjubybot Date: Fri, 21 Feb 2025 15:38:54 +0100 Subject: [PATCH] batch users from csv --- .../lib/codebattle/utils/populate_users.ex | 13 ++++++++----- .../test/codebattle/utils/populate_users_test.exs | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/services/app/apps/codebattle/lib/codebattle/utils/populate_users.ex b/services/app/apps/codebattle/lib/codebattle/utils/populate_users.ex index e7e06ec66..848ec918d 100644 --- a/services/app/apps/codebattle/lib/codebattle/utils/populate_users.ex +++ b/services/app/apps/codebattle/lib/codebattle/utils/populate_users.ex @@ -4,12 +4,15 @@ defmodule Codebattle.Utils.PopulateUsers do def from_csv(file) do utc_now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) - users = - File.stream!(file) - |> NimbleCSV.RFC4180.parse_stream() - |> Stream.map(&row_to_user(&1, utc_now)) - |> Enum.to_list() + File.stream!(file) + |> NimbleCSV.RFC4180.parse_stream() + |> Stream.chunk_every(500) + |> Stream.each(&process_batch(&1, utc_now)) + |> Stream.run() + end + defp process_batch(users, now) do + users = Enum.map(users, &row_to_user(&1, now)) Codebattle.Repo.insert_all(Codebattle.User, users) end diff --git a/services/app/apps/codebattle/test/codebattle/utils/populate_users_test.exs b/services/app/apps/codebattle/test/codebattle/utils/populate_users_test.exs index 8b57506f4..0355662bb 100644 --- a/services/app/apps/codebattle/test/codebattle/utils/populate_users_test.exs +++ b/services/app/apps/codebattle/test/codebattle/utils/populate_users_test.exs @@ -14,7 +14,7 @@ defmodule Codebattle.Utils.PopulateUsersTest do IO.write(fd, csv) File.close(fd) - assert {4, nil} = Codebattle.Utils.PopulateUsers.from_csv(path) + assert :ok = Codebattle.Utils.PopulateUsers.from_csv(path) assert %{name: "user1"} = Codebattle.User.authenticate("user1", "p@ssw0rd!") end end