From ea3ee0a32a57ac5ce0f6ed0d6946e245862ab461 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:22:59 -0500 Subject: [PATCH 01/18] Add PackageCompiler.jl to PySR project --- pysr/julia_helpers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 2eafa67c6..e82fc50f1 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -234,10 +234,17 @@ def _add_sr_to_julia_project(Main, io_arg): ) Main.clustermanagers_spec = Main.PackageSpec( name="ClusterManagers", - url="https://github.com/JuliaParallel/ClusterManagers.jl", - rev="14e7302f068794099344d5d93f71979aaf4fbeb3", + rev="v0.4.2", + ) + Main.packagecompiler_spec = Main.PackageSpec( + name="PackageCompiler", + rev="v2.1.0", + ) + Main.eval( + "Pkg.add([" + + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) + + f"], {io_arg})" ) - Main.eval(f"Pkg.add([sr_spec, clustermanagers_spec], {io_arg})") def _escape_filename(filename): From 0b8d933d6996c384cc35acee546e772d91e2a1d7 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:23:14 -0500 Subject: [PATCH 02/18] Add function to create simple sysimage --- pysr/package_compiler.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 pysr/package_compiler.py diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py new file mode 100644 index 000000000..f7a5b92b5 --- /dev/null +++ b/pysr/package_compiler.py @@ -0,0 +1,36 @@ +"""Functions to create a sysimage for PySR.""" + +from pathlib import Path +import numpy as np + +from . import PySRRegressor +from .julia_helpers import init_julia + + +def create_sysimage(): + """Create a PackageCompiler.jl sysimage from a simple PySR run.""" + sysimage_name = "pysr.so" + + # Example dataset: + X = 2 * np.random.randn(100, 5) + y = 2.5382 * np.cos(X[:, 3]) + X[:, 0] ** 2 - 0.5 + + model = PySRRegressor( + unary_operators=["cos"], + procs=8, + niterations=2, + populations=4, + verbosity=0, + progress=False, + ) + model.fit(X, y) + + from julia import Main + + cur_project_dir = Main.eval("splitdir(dirname(Base.active_project()))[1]") + from julia import PackageCompiler + + PackageCompiler.create_sysimage( + ["SymbolicRegression"], + sysimage_path=str(Path(cur_project_dir) / sysimage_name), + ) From 9758fa1aa0ec7cf2cba4b4d0aa9089ed4fd053e7 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:23:27 -0500 Subject: [PATCH 03/18] Add `create_sysimage` to `__init__` --- pysr/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysr/__init__.py b/pysr/__init__.py index e303becb2..f6e5c9894 100644 --- a/pysr/__init__.py +++ b/pysr/__init__.py @@ -11,3 +11,4 @@ from .feynman_problems import Problem, FeynmanProblem from .export_jax import sympy2jax from .export_torch import sympy2torch +from .package_compiler import create_sysimage From 071adbdcd453192866f832b43ce594b395d0795e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 22 Nov 2022 18:26:49 -0500 Subject: [PATCH 04/18] Put `pysr.so` into correct directory --- pysr/package_compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index f7a5b92b5..4058b9292 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -27,7 +27,7 @@ def create_sysimage(): from julia import Main - cur_project_dir = Main.eval("splitdir(dirname(Base.active_project()))[1]") + cur_project_dir = Main.eval("dirname(Base.active_project())") from julia import PackageCompiler PackageCompiler.create_sysimage( From fe81758071e9b7f14efa9fb2b33f20f79e119d55 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:26:37 -0500 Subject: [PATCH 05/18] Bump SR version with improved precompilation --- pysr/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysr/version.py b/pysr/version.py index 7a2c5b6c1..2f5955ae4 100644 --- a/pysr/version.py +++ b/pysr/version.py @@ -1,2 +1,2 @@ __version__ = "0.11.11" -__symbolic_regression_jl_version__ = "0.14.4" +__symbolic_regression_jl_version__ = "0.14.5" From 724a901d11024868e6a713c8f7bb82f545eea0fb Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:27:05 -0500 Subject: [PATCH 06/18] Add option to create sysimage to `install()` --- pysr/julia_helpers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index e82fc50f1..884dc4034 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -7,6 +7,7 @@ from julia.api import JuliaError from .version import __version__, __symbolic_regression_jl_version__ +from .package_compiler import create_sysimage juliainfo = None julia_initialized = False @@ -65,7 +66,7 @@ def _get_io_arg(quiet): return io_arg -def install(julia_project=None, quiet=False): # pragma: no cover +def install(julia_project=None, quiet=False, compile=False): # pragma: no cover """ Install PyCall.jl and all required dependencies for SymbolicRegression.jl. @@ -89,6 +90,9 @@ def install(julia_project=None, quiet=False): # pragma: no cover Main.eval("using Pkg") Main.eval(f"Pkg.instantiate({io_arg})") Main.eval(f"Pkg.precompile({io_arg})") + if compile: + create_sysimage(julia_project=julia_project, quiet=quiet) + if not quiet: warnings.warn( "It is recommended to restart Python after installing PySR's dependencies," From b4c4f8af537cdaca0644a702832b99a7e64e19fa Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:27:21 -0500 Subject: [PATCH 07/18] Clean up sysimage creation --- pysr/package_compiler.py | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 4058b9292..22cd4b261 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -7,30 +7,12 @@ from .julia_helpers import init_julia -def create_sysimage(): +def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): """Create a PackageCompiler.jl sysimage from a simple PySR run.""" - sysimage_name = "pysr.so" - - # Example dataset: - X = 2 * np.random.randn(100, 5) - y = 2.5382 * np.cos(X[:, 3]) + X[:, 0] ** 2 - 0.5 - - model = PySRRegressor( - unary_operators=["cos"], - procs=8, - niterations=2, - populations=4, - verbosity=0, - progress=False, - ) - model.fit(X, y) - - from julia import Main + Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") + sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler - PackageCompiler.create_sysimage( - ["SymbolicRegression"], - sysimage_path=str(Path(cur_project_dir) / sysimage_name), - ) + PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) From e0e1247a49a7367111258cd2f2db1a553a56ccef Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:33:52 -0500 Subject: [PATCH 08/18] Move installation utilities to compilation module --- pysr/__init__.py | 2 +- pysr/julia_helpers.py | 68 -------------------------------- pysr/package_compiler.py | 83 ++++++++++++++++++++++++++++++++++++++-- pysr/sr.py | 2 +- 4 files changed, 81 insertions(+), 74 deletions(-) diff --git a/pysr/__init__.py b/pysr/__init__.py index f6e5c9894..2b7c64601 100644 --- a/pysr/__init__.py +++ b/pysr/__init__.py @@ -7,7 +7,7 @@ best_callable, best_row, ) -from .julia_helpers import install +from .package_compiler import install from .feynman_problems import Problem, FeynmanProblem from .export_jax import sympy2jax from .export_torch import sympy2torch diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 884dc4034..af6f515a1 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -7,7 +7,6 @@ from julia.api import JuliaError from .version import __version__, __symbolic_regression_jl_version__ -from .package_compiler import create_sysimage juliainfo = None julia_initialized = False @@ -66,40 +65,6 @@ def _get_io_arg(quiet): return io_arg -def install(julia_project=None, quiet=False, compile=False): # pragma: no cover - """ - Install PyCall.jl and all required dependencies for SymbolicRegression.jl. - - Also updates the local Julia registry. - """ - import julia - - _julia_version_assertion() - # Set JULIA_PROJECT so that we install in the pysr environment - processed_julia_project, is_shared = _process_julia_project(julia_project) - _set_julia_project_env(processed_julia_project, is_shared) - - julia.install(quiet=quiet) - Main = init_julia(julia_project, quiet=quiet) - io_arg = _get_io_arg(quiet) - - if is_shared: - # Install SymbolicRegression.jl: - _add_sr_to_julia_project(Main, io_arg) - - Main.eval("using Pkg") - Main.eval(f"Pkg.instantiate({io_arg})") - Main.eval(f"Pkg.precompile({io_arg})") - if compile: - create_sysimage(julia_project=julia_project, quiet=quiet) - - if not quiet: - warnings.warn( - "It is recommended to restart Python after installing PySR's dependencies," - " so that the Julia environment is properly initialized." - ) - - def _import_error(): return """ Required dependencies are not installed or built. Run the following code in the Python REPL: @@ -229,28 +194,6 @@ def init_julia(julia_project=None, quiet=False, julia_kwargs=None): return Main -def _add_sr_to_julia_project(Main, io_arg): - Main.eval("using Pkg") - Main.sr_spec = Main.PackageSpec( - name="SymbolicRegression", - url="https://github.com/MilesCranmer/SymbolicRegression.jl", - rev="v" + __symbolic_regression_jl_version__, - ) - Main.clustermanagers_spec = Main.PackageSpec( - name="ClusterManagers", - rev="v0.4.2", - ) - Main.packagecompiler_spec = Main.PackageSpec( - name="PackageCompiler", - rev="v2.1.0", - ) - Main.eval( - "Pkg.add([" - + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) - + f"], {io_arg})" - ) - - def _escape_filename(filename): """Turn a path into a string with correctly escaped backslashes.""" str_repr = str(filename) @@ -291,17 +234,6 @@ def _load_cluster_manager(Main, cluster_manager): Main.eval(f"import ClusterManagers: addprocs_{cluster_manager}") return Main.eval(f"addprocs_{cluster_manager}") - -def _update_julia_project(Main, is_shared, io_arg): - try: - if is_shared: - _add_sr_to_julia_project(Main, io_arg) - Main.eval("using Pkg") - Main.eval(f"Pkg.resolve({io_arg})") - except (JuliaError, RuntimeError) as e: - raise ImportError(_import_error()) from e - - def _load_backend(Main): try: # Load namespace, so that various internal operators work: diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 22cd4b261..f2cc5a41a 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -1,15 +1,90 @@ """Functions to create a sysimage for PySR.""" from pathlib import Path +import warnings + import numpy as np +from julia.api import JuliaError -from . import PySRRegressor -from .julia_helpers import init_julia +from .version import __symbolic_regression_jl_version__ +from .julia_helpers import ( + init_julia, + _julia_version_assertion, + _set_julia_project_env, + _get_io_arg, + _process_julia_project, + _import_error, +) -def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): - """Create a PackageCompiler.jl sysimage from a simple PySR run.""" +def _add_sr_to_julia_project(Main, io_arg): + Main.eval("using Pkg") + Main.sr_spec = Main.PackageSpec( + name="SymbolicRegression", + url="https://github.com/MilesCranmer/SymbolicRegression.jl", + rev="v" + __symbolic_regression_jl_version__, + ) + Main.clustermanagers_spec = Main.PackageSpec( + name="ClusterManagers", + rev="v0.4.2", + ) + Main.packagecompiler_spec = Main.PackageSpec( + name="PackageCompiler", + rev="v2.1.0", + ) + Main.eval( + "Pkg.add([" + + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) + + f"], {io_arg})" + ) + + +def _update_julia_project(Main, is_shared, io_arg): + try: + if is_shared: + _add_sr_to_julia_project(Main, io_arg) + Main.eval("using Pkg") + Main.eval(f"Pkg.resolve({io_arg})") + except (JuliaError, RuntimeError) as e: + raise ImportError(_import_error()) from e + + +def install(julia_project=None, quiet=False, compile=False): # pragma: no cover + """ + Install PyCall.jl and all required dependencies for SymbolicRegression.jl. + + Also updates the local Julia registry. + """ + import julia + _julia_version_assertion() + # Set JULIA_PROJECT so that we install in the pysr environment + processed_julia_project, is_shared = _process_julia_project(julia_project) + _set_julia_project_env(processed_julia_project, is_shared) + + julia.install(quiet=quiet) + Main = init_julia(julia_project, quiet=quiet) + io_arg = _get_io_arg(quiet) + + if is_shared: + # Install SymbolicRegression.jl: + _add_sr_to_julia_project(Main, io_arg) + + Main.eval("using Pkg") + Main.eval(f"Pkg.instantiate({io_arg})") + Main.eval(f"Pkg.precompile({io_arg})") + if compile: + create_sysimage(julia_project=julia_project, quiet=quiet) + + if not quiet: + warnings.warn( + "It is recommended to restart Python after installing PySR's dependencies," + " so that the Julia environment is properly initialized." + ) + + +def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): + """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) diff --git a/pysr/sr.py b/pysr/sr.py index 2238e6420..a6786a581 100644 --- a/pysr/sr.py +++ b/pysr/sr.py @@ -21,13 +21,13 @@ check_is_fitted, ) +from .package_compiler import _update_julia_project from .julia_helpers import ( init_julia, _process_julia_project, is_julia_version_greater_eq, _escape_filename, _load_cluster_manager, - _update_julia_project, _load_backend, ) from .export_numpy import CallableEquation From 51b1e8b7661df14223750315b998502261114187 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:38:56 -0500 Subject: [PATCH 09/18] Separate install and compile steps --- pysr/__init__.py | 3 +-- pysr/package_compiler.py | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pysr/__init__.py b/pysr/__init__.py index 2b7c64601..96365bf88 100644 --- a/pysr/__init__.py +++ b/pysr/__init__.py @@ -7,8 +7,7 @@ best_callable, best_row, ) -from .package_compiler import install +from .package_compiler import install, compile from .feynman_problems import Problem, FeynmanProblem from .export_jax import sympy2jax from .export_torch import sympy2torch -from .package_compiler import create_sysimage diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index f2cc5a41a..130c520d7 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -49,7 +49,7 @@ def _update_julia_project(Main, is_shared, io_arg): raise ImportError(_import_error()) from e -def install(julia_project=None, quiet=False, compile=False): # pragma: no cover +def install(julia_project=None, quiet=False): # pragma: no cover """ Install PyCall.jl and all required dependencies for SymbolicRegression.jl. @@ -73,8 +73,6 @@ def install(julia_project=None, quiet=False, compile=False): # pragma: no cover Main.eval("using Pkg") Main.eval(f"Pkg.instantiate({io_arg})") Main.eval(f"Pkg.precompile({io_arg})") - if compile: - create_sysimage(julia_project=julia_project, quiet=quiet) if not quiet: warnings.warn( @@ -83,11 +81,12 @@ def install(julia_project=None, quiet=False, compile=False): # pragma: no cover ) -def create_sysimage(sysimage_name="pysr.so", julia_project=None, quiet=False): +def compile(sysimage_name="pysr.so", julia_project=None, quiet=False): """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler + from julia import SymbolicRegression PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) From 28ef2415741c3eac3c9970d939ab05bfeaea573b Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 12:39:06 -0500 Subject: [PATCH 10/18] Bump PySR version with compilation --- pysr/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysr/version.py b/pysr/version.py index 2f5955ae4..df1f4afe4 100644 --- a/pysr/version.py +++ b/pysr/version.py @@ -1,2 +1,2 @@ -__version__ = "0.11.11" +__version__ = "0.11.12" __symbolic_regression_jl_version__ = "0.14.5" From d1c78a33ffe8a034ef9df9d557e9236b3add512a Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 14:07:29 -0500 Subject: [PATCH 11/18] Working precompilation --- pysr/julia_helpers.py | 48 +++++++++++++++++++++++++++++++++++----- pysr/package_compiler.py | 9 ++++++-- pysr/test/test_env.py | 3 ++- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index af6f515a1..d1801b8f6 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -32,6 +32,27 @@ def _load_juliainfo(): return juliainfo +def _get_julia_project_dir(): + # Assumes it is in JULIA_PROJECT: + assert "JULIA_PROJECT" in os.environ and os.environ["JULIA_PROJECT"] != "" + try: + cmds = [ + "julia", + "-e using Pkg; print(Pkg.project().path)", + ] + julia_project_dir_str = subprocess.run( + cmds, + capture_output=True, + env=os.environ, + ).stdout.decode() + except FileNotFoundError: + env_path = os.environ["PATH"] + raise FileNotFoundError( + f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}", + ) + return Path(julia_project_dir_str).parent + + def _get_julia_env_dir(): # Have to manually get env dir: try: @@ -114,7 +135,7 @@ def _check_for_conflicting_libraries(): # pragma: no cover ) -def init_julia(julia_project=None, quiet=False, julia_kwargs=None): +def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs=None): """Initialize julia binary, turning off compiled modules if needed.""" global julia_initialized global julia_kwargs_at_initialization @@ -123,16 +144,32 @@ def init_julia(julia_project=None, quiet=False, julia_kwargs=None): if not julia_initialized: _check_for_conflicting_libraries() + _julia_version_assertion() + processed_julia_project, is_shared = _process_julia_project(julia_project) + _set_julia_project_env(processed_julia_project, is_shared) + + # TODO: Make checking optional. + # Check if sysimage exists: + if sysimage_name is None: + # TODO: Is there a faster way to get this dir? + expected_sysimage = _get_julia_project_dir() / "pysr.so" + print(f"Checking for sysimage at {expected_sysimage}") + # Check if this file exists: + if expected_sysimage.exists(): + sysimage_name = str(expected_sysimage) + if julia_kwargs is None: julia_kwargs = {"optimize": 3} - from julia.core import JuliaInfo, UnsupportedPythonError + if sysimage_name is not None and "sysimage" not in julia_kwargs: + sysimage = str(sysimage_name) + print(f"Found existing sysimage at {sysimage}. Loading.") + julia_kwargs["sysimage"] = sysimage_name - _julia_version_assertion() - processed_julia_project, is_shared = _process_julia_project(julia_project) - _set_julia_project_env(processed_julia_project, is_shared) + from julia.core import JuliaInfo, UnsupportedPythonError try: + # TODO: Can we just get env info from this? info = JuliaInfo.load(julia="julia") except FileNotFoundError: env_path = os.environ["PATH"] @@ -234,6 +271,7 @@ def _load_cluster_manager(Main, cluster_manager): Main.eval(f"import ClusterManagers: addprocs_{cluster_manager}") return Main.eval(f"addprocs_{cluster_manager}") + def _load_backend(Main): try: # Load namespace, so that various internal operators work: diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 130c520d7..91892c4a9 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -81,12 +81,17 @@ def install(julia_project=None, quiet=False): # pragma: no cover ) -def compile(sysimage_name="pysr.so", julia_project=None, quiet=False): +def compile( + julia_project=None, + quiet=False, + sysimage_name="pysr.so", +): """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" Main = init_julia(julia_project=julia_project, quiet=quiet) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler - from julia import SymbolicRegression + + Main.eval("using SymbolicRegression") PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) diff --git a/pysr/test/test_env.py b/pysr/test/test_env.py index c044cfd0c..0d18412ad 100644 --- a/pysr/test/test_env.py +++ b/pysr/test/test_env.py @@ -5,6 +5,7 @@ from tempfile import TemporaryDirectory from .. import julia_helpers +from .. import package_compiler class TestJuliaProject(unittest.TestCase): @@ -29,7 +30,7 @@ def test_custom_shared_env(self): f'pushfirst!(DEPOT_PATH, "{julia_helpers._escape_filename(tmpdir)}")' ) test_env_name = "@pysr_test_env" - julia_helpers.install(julia_project=test_env_name) + package_compiler.install(julia_project=test_env_name) Main = julia_helpers.init_julia(julia_project=test_env_name) # Try to use env: From f84f5a0be5444a9f448840e640f397c2b2550984 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 14:23:01 -0500 Subject: [PATCH 12/18] Only try loading sysimage if Julia uninitialized --- pysr/julia_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index d1801b8f6..454aa805e 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -150,7 +150,7 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs # TODO: Make checking optional. # Check if sysimage exists: - if sysimage_name is None: + if sysimage_name is None and not julia_initialized: # TODO: Is there a faster way to get this dir? expected_sysimage = _get_julia_project_dir() / "pysr.so" print(f"Checking for sysimage at {expected_sysimage}") @@ -161,7 +161,7 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs if julia_kwargs is None: julia_kwargs = {"optimize": 3} - if sysimage_name is not None and "sysimage" not in julia_kwargs: + if sysimage_name is not None and "sysimage" not in julia_kwargs and not julia_initialized: sysimage = str(sysimage_name) print(f"Found existing sysimage at {sysimage}. Loading.") julia_kwargs["sysimage"] = sysimage_name From 48fd3fbe02b707ee4d118d0666ca20185c21421e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 28 Nov 2022 14:23:25 -0500 Subject: [PATCH 13/18] Clean up sysimage check --- pysr/julia_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 454aa805e..4ab4a6a16 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -153,7 +153,6 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs if sysimage_name is None and not julia_initialized: # TODO: Is there a faster way to get this dir? expected_sysimage = _get_julia_project_dir() / "pysr.so" - print(f"Checking for sysimage at {expected_sysimage}") # Check if this file exists: if expected_sysimage.exists(): sysimage_name = str(expected_sysimage) From 78e7e22d57827882e8cc584fc11f76e1f476f313 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 19:23:39 +0000 Subject: [PATCH 14/18] Format code with black --- pysr/julia_helpers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 4ab4a6a16..baea787b3 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -160,7 +160,11 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs if julia_kwargs is None: julia_kwargs = {"optimize": 3} - if sysimage_name is not None and "sysimage" not in julia_kwargs and not julia_initialized: + if ( + sysimage_name is not None + and "sysimage" not in julia_kwargs + and not julia_initialized + ): sysimage = str(sysimage_name) print(f"Found existing sysimage at {sysimage}. Loading.") julia_kwargs["sysimage"] = sysimage_name From 4256604634c09ea3a96c164c657cc46d870466cd Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 09:32:31 -0500 Subject: [PATCH 15/18] Ensure that we compile an optimized image --- pysr/package_compiler.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 91892c4a9..02deef458 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -32,11 +32,18 @@ def _add_sr_to_julia_project(Main, io_arg): name="PackageCompiler", rev="v2.1.0", ) + Main.pycall_spec = Main.PackageSpec( + name="PyCall", + rev="v1.94.1", + ) Main.eval( "Pkg.add([" - + ", ".join(["sr_spec", "clustermanagers_spec", "packagecompiler_spec"]) + + ", ".join( + ["sr_spec", "clustermanagers_spec", "packagecompiler_spec", "pycall_spec"] + ) + f"], {io_arg})" ) + Main.eval(f'Pkg.build("PyCall", {io_arg})') def _update_julia_project(Main, is_shared, io_arg): @@ -87,11 +94,23 @@ def compile( sysimage_name="pysr.so", ): """Create a PackageCompiler.jl sysimage for SymbolicRegression.jl.""" - Main = init_julia(julia_project=julia_project, quiet=quiet) + Main = init_julia( + julia_project=julia_project, + quiet=quiet, + julia_kwargs={ + "compiled_modules": False, + "optimize": 3, + "threads": "auto", + "compile": "all", + }, + ) cur_project_dir = Main.eval("dirname(Base.active_project())") sysimage_path = str(Path(cur_project_dir) / sysimage_name) from julia import PackageCompiler + Main.eval("using PyCall") Main.eval("using SymbolicRegression") - PackageCompiler.create_sysimage(["SymbolicRegression"], sysimage_path=sysimage_path) + PackageCompiler.create_sysimage( + ["SymbolicRegression", "PyCall"], sysimage_path=sysimage_path + ) From 0ccca86e85807ae4efee8cfbe902de326602b53e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 10:04:08 -0500 Subject: [PATCH 16/18] Improve time to check project dir --- pysr/julia_helpers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index baea787b3..3833cb8fe 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -38,7 +38,11 @@ def _get_julia_project_dir(): try: cmds = [ "julia", - "-e using Pkg; print(Pkg.project().path)", + "--compile=min", + "--startup-file=no", + "-O0", + "-g0", + "-e import Pkg: project; print(project().path)", ] julia_project_dir_str = subprocess.run( cmds, @@ -56,8 +60,16 @@ def _get_julia_project_dir(): def _get_julia_env_dir(): # Have to manually get env dir: try: + cmds = [ + "julia", + "--compile=min", + "--startup-file=no", + "-O0", + "-g0", + "-e import Pkg: envdir; print(envdir())", + ] julia_env_dir_str = subprocess.run( - ["julia", "-e using Pkg; print(Pkg.envdir())"], + cmds, capture_output=True, env=os.environ, ).stdout.decode() From aca0e4b32030ed9b70063a7a306b7c43eb6b4b60 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 10:12:38 -0500 Subject: [PATCH 17/18] Ensure that compiling twice works --- pysr/package_compiler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysr/package_compiler.py b/pysr/package_compiler.py index 02deef458..e7537d499 100644 --- a/pysr/package_compiler.py +++ b/pysr/package_compiler.py @@ -2,6 +2,7 @@ from pathlib import Path import warnings +from multiprocessing import cpu_count import numpy as np from julia.api import JuliaError @@ -97,10 +98,11 @@ def compile( Main = init_julia( julia_project=julia_project, quiet=quiet, + use_sysimage=False, julia_kwargs={ "compiled_modules": False, "optimize": 3, - "threads": "auto", + "threads": cpu_count(), "compile": "all", }, ) From dc2fef561cb691850436d87d66fe07310f96d014 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Tue, 29 Nov 2022 10:12:50 -0500 Subject: [PATCH 18/18] Allow forced init without sysimage --- pysr/julia_helpers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pysr/julia_helpers.py b/pysr/julia_helpers.py index 3833cb8fe..8c32ca0f7 100644 --- a/pysr/julia_helpers.py +++ b/pysr/julia_helpers.py @@ -147,7 +147,13 @@ def _check_for_conflicting_libraries(): # pragma: no cover ) -def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs=None): +def init_julia( + julia_project=None, + quiet=False, + use_sysimage=True, + sysimage_name=None, + julia_kwargs=None, +): """Initialize julia binary, turning off compiled modules if needed.""" global julia_initialized global julia_kwargs_at_initialization @@ -162,7 +168,7 @@ def init_julia(julia_project=None, quiet=False, sysimage_name=None, julia_kwargs # TODO: Make checking optional. # Check if sysimage exists: - if sysimage_name is None and not julia_initialized: + if use_sysimage and sysimage_name is None and not julia_initialized: # TODO: Is there a faster way to get this dir? expected_sysimage = _get_julia_project_dir() / "pysr.so" # Check if this file exists: