From 83dfb5d68ce76a7f29a862531541a8aa1856d011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Sat, 25 Jan 2025 14:02:48 +0100 Subject: [PATCH 1/2] Feedback --- README.md | 14 ++++++++++++++ lib/pythonx/application.ex | 9 ++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d348df1..ebc83d1 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,20 @@ Pythonx runs a Python interpreter in the same OS process as your Elixir application, allowing you to evaluate Python code and conveniently convert between Python and Elixir data structures. +The goal of this project is to better integrate Python workflows within +Livebook and its usage in actual projects must be done with care due to +Python's global interpreter lock (GIL), which prevents from multiple threads +executing Python code at the same time. Consequently, calling `Pythonx` +from multiple Elixir processes does not provide the concurrency you might +expect and thus it can be a source of bottlenecks. However, this concerns +regular Python code. Packages with CPU-intense functionality, such as `numpy`, +have native implementation of many functions and invoking those releases the +GIL. GIL is also released when waiting on I/O operations. In other words, +if you are using this library to integrate with Python, make sure it happens +in a single Elixir process or that its underlying libraries can deal with +concurrent invocation. Otherwqise, prefer to use Elixir's `System.cmd/3` or +`Port`s to manage multiple Python programs via I/O. + ## Usage (script) Add Pythonx to your dependencies: diff --git a/lib/pythonx/application.ex b/lib/pythonx/application.ex index f235d60..84260ae 100644 --- a/lib/pythonx/application.ex +++ b/lib/pythonx/application.ex @@ -20,14 +20,13 @@ defmodule Pythonx.Application do # If configured, Python and dependencies are fetched at compile time, # so we automatically initialize the interpreter on boot. + # + # TODO: My suggestion would be to call ut :pythonx, :uv_init, to make + # it clear it will start the runtime. if pyproject_toml = Application.compile_env(:pythonx, :uv)[:pyproject_toml] do + Pythonx.Uv.fetch(pyproject_toml, true) defp maybe_uv_init(), do: Pythonx.Uv.init(unquote(pyproject_toml), true) else defp maybe_uv_init(), do: :noop end end - -# If configured, fetch Python and dependencies when compiling. -if pyproject_toml = Application.compile_env(:pythonx, :uv)[:pyproject_toml] do - Pythonx.Uv.fetch(pyproject_toml, true) -end From 8827e8384f7725a988d89d14a5630671cd9bd48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Sat, 25 Jan 2025 20:21:11 +0700 Subject: [PATCH 2/2] up --- README.md | 2 +- lib/pythonx/application.ex | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ebc83d1..5ec70b2 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Configure the desired Python version and dependencies in your ```elixir import Config -config :pythonx, :uv, +config :pythonx, :uv_init, pyproject_toml: """ [project] name = "project" diff --git a/lib/pythonx/application.ex b/lib/pythonx/application.ex index 84260ae..a4b8053 100644 --- a/lib/pythonx/application.ex +++ b/lib/pythonx/application.ex @@ -18,12 +18,9 @@ defmodule Pythonx.Application do end end - # If configured, Python and dependencies are fetched at compile time, - # so we automatically initialize the interpreter on boot. - # - # TODO: My suggestion would be to call ut :pythonx, :uv_init, to make - # it clear it will start the runtime. - if pyproject_toml = Application.compile_env(:pythonx, :uv)[:pyproject_toml] do + # If configured, we fetch Python and dependencies at compile time + # and we automatically initialize the interpreter on boot. + if pyproject_toml = Application.compile_env(:pythonx, :uv_init)[:pyproject_toml] do Pythonx.Uv.fetch(pyproject_toml, true) defp maybe_uv_init(), do: Pythonx.Uv.init(unquote(pyproject_toml), true) else