From 489614391c2601248cc8f3262a4d16ae760a889f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Tue, 20 May 2025 23:20:33 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=90=9B=20Handle=20Conda=20file=20base?= =?UTF-8?q?d=20dependencies=20in=20metadata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes issue where the file type dependency obtained using `pip freeze` causes issues with the python environment metadata. --- simvue/metadata.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/simvue/metadata.py b/simvue/metadata.py index 0f63981c..b5dcca0a 100644 --- a/simvue/metadata.py +++ b/simvue/metadata.py @@ -78,7 +78,7 @@ def git_info(repository: str) -> dict[str, typing.Any]: def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]: """Retrieve a dictionary of Python dependencies if lock file is available""" - python_meta: dict[str, str] = {} + python_meta: dict[str, dict] = {} if (pyproject_file := pathlib.Path(repository).joinpath("pyproject.toml")).exists(): content = toml.load(pyproject_file) @@ -107,18 +107,27 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]: with contextlib.suppress((KeyError, ImportError)): from pip._internal.operations.freeze import freeze - python_meta["environment"] = { - entry[0]: entry[-1] - for line in freeze(local_only=True) - if (entry := line.split("==")) - } + # Conda supports having file names with @ as entries + # in the requirements.txt file as opposed to == + python_meta["environment"] = {} + + for line in freeze(local_only=True): + if line.startswith("-e"): + python_meta["environment"]["local_install"] = line.split(" ")[-1] + continue + if "@" in line: + entry = line.split("@") + python_meta["environment"][entry[0].strip()] = entry[-1].strip() + elif "==" in line: + entry = line.split("==") + python_meta["environment"][entry[0].strip()] = entry[-1].strip() return python_meta def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]: """Retrieve a dictionary of Rust dependencies if lock file available""" - rust_meta: dict[str, str] = {} + rust_meta: dict[str, dict] = {} if (cargo_file := pathlib.Path(repository).joinpath("Cargo.toml")).exists(): content = toml.load(cargo_file).get("package", {}) @@ -134,7 +143,7 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]: cargo_dat = toml.load(cargo_lock) rust_meta["environment"] = { dependency["name"]: dependency["version"] - for dependency in cargo_dat.get("package") + for dependency in cargo_dat.get("package", []) } return rust_meta @@ -142,7 +151,7 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]: def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]: """Retrieve a dictionary of Julia dependencies if a project file is available""" - julia_meta: dict[str, str] = {} + julia_meta: dict[str, dict] = {} if (project_file := pathlib.Path(repository).joinpath("Project.toml")).exists(): content = toml.load(project_file) julia_meta["project"] = { @@ -155,7 +164,7 @@ def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]: def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]: - js_meta: dict[str, str] = {} + js_meta: dict[str, dict] = {} if ( project_file := pathlib.Path(repository).joinpath("package-lock.json") ).exists(): From 98672dcefb05e81620f368c1b54378a53b0304e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 8 Aug 2025 09:35:25 +0100 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Improve=20Conda=20envi?= =?UTF-8?q?ronment=20file=20handling.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 308 +++++++++--------- poetry.lock | 65 +++- pyproject.toml | 3 +- simvue/metadata.py | 64 ++++ .../example_data/python_conda/environment.yml | 40 +++ tests/unit/test_metadata.py | 7 +- 6 files changed, 336 insertions(+), 151 deletions(-) create mode 100644 tests/example_data/python_conda/environment.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index b5739350..1eb91a4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,263 +1,277 @@ -# Change log +# Change Log + +## Unreleased + +- Improve handling of Conda based environments in metadata collection. + ## [v2.1.2](https://github.com/simvue-io/client/releases/tag/v2.1.2) - 2025-06-25 -* Fixed issue in downloading files from tenant runs. -* Fixed bug in pagination whereby the count value specified by the user is ignored. -* Fixed bug where uploading larger files timed out leading to file of size 0B. -* Fixed bug where if the range or threshold of an alert is zero the alert type validation fails. -* Fixed bug in `Folder.ids` where `kwargs` were not being passed to `GET`. -* Ensured all threads have `daemon=True` to prevent hanging on termination. -* Added error when `close()` method is called within the `simvue.Run` context manager. + +- Fixed issue in downloading files from tenant runs. +- Fixed bug in pagination whereby the count value specified by the user is ignored. +- Fixed bug where uploading larger files timed out leading to file of size 0B. +- Fixed bug where if the range or threshold of an alert is zero the alert type validation fails. +- Fixed bug in `Folder.ids` where `kwargs` were not being passed to `GET`. +- Ensured all threads have `daemon=True` to prevent hanging on termination. +- Added error when `close()` method is called within the `simvue.Run` context manager. + ## [v2.1.1](https://github.com/simvue-io/client/releases/tag/v2.1.1) - 2025-04-25 -* Changed from CO2 Signal to ElectricityMaps -* Fixed a number of bugs in how offline mode is handled with emissions -* Streamlined EmissionsMonitor class and handling -* Fixed bugs in client getting results from Simvue server arising from pagination -* Fixed bug in setting visibility in `run.init` method -* Default setting in `Client.get_runs` is now `show_shared=True` + +- Changed from CO2 Signal to ElectricityMaps +- Fixed a number of bugs in how offline mode is handled with emissions +- Streamlined EmissionsMonitor class and handling +- Fixed bugs in client getting results from Simvue server arising from pagination +- Fixed bug in setting visibility in `run.init` method +- Default setting in `Client.get_runs` is now `show_shared=True` + ## [v2.1.0](https://github.com/simvue-io/client/releases/tag/v2.1.0) - 2025-03-28 -* Removed CodeCarbon dependence in favour of a slimmer solution using the CO2 Signal API. -* Added sorting to server queries, users can now specify to sort by columns during data retrieval from the database. -* Added pagination of results from server to reduce await time in responses. -* Added equivalent of folder details modification function to `Client` class. + +- Removed CodeCarbon dependence in favour of a slimmer solution using the CO2 Signal API. +- Added sorting to server queries, users can now specify to sort by columns during data retrieval from the database. +- Added pagination of results from server to reduce await time in responses. +- Added equivalent of folder details modification function to `Client` class. + ## [v2.0.1](https://github.com/simvue-io/client/releases/tag/v2.0.1) - 2025-03-24 -* Improvements to docstrings on methods, classes and functions. + +- Improvements to docstrings on methods, classes and functions. + ## [v2.0.0](https://github.com/simvue-io/client/releases/tag/v2.0.0) - 2025-03-07 -* Add new example notebooks -* Update and refactor examples to work with v2.0 -* Fix bug in offline artifacts using wrong file path -* Change names of sustainability metrics -* Fix `Self` being used in typing Generators so that Simvue works with Python 3.10 in Conda -* Updated codecarbon to work with new API -* Codecarbon now works with offline mode -* Codecarbon metadata dict is now nested -* Add PID to sender lock file so it can recover from crashes -* Add accept Gzip encoding -* Fixed list of processes to add / remove from existing list of objects -* Add step to resource metrics -* Fix bug where process user alerts should not be overridden if manually set by the user -* Removed 'no config file' and 'unstaged changes' warnings from Offline mode as they do not apply -* Made `staging_check` not apply in Offline mode -* Added heartbeat functionality to Offline mode -* Moved away from `FlatDict` module for metadata collection, fixes Simvue in Jupyter notebooks -* Fixed `reconnect()` by setting `read_only` to False and added tests -* Fixed resource metrics collection to return measurement on startup and use short interval for more accurate measurements -* Fixed `set_pid` so that resource metrics are also collected for child processes of it -* Improved sender by having all cached files read at start and lock file so only one sender runs at once -* Added `name` option in `log_alert` and added tests -* Fixed client `get_alerts` and improved tests -* Removed all server config checks in Offline mode -* Fixed `add_alerts` so that it now works with both IDs and names -* Improved alert and folder deduplication methods to rely on 409 responses from server upon creation -* Added `attach_to_run` option to create alerts methods so that alerts can be created without a run attached -* Improved merging of local staging file and _staged dict using `deepmerge` - fixes bugs with tags, alerts and metadata in offline mode -* Added `started`, `created` and `ended` timestamps to runs in offline mode -* Remove all erronous server calls in offline mode -* Fixed method to find simvue.toml config files, now just looks in cwd and home -* Added run notification option to `run.init` so that users can now get emails upon their runs completing -* Fixed artifact retrieval by run so that `category` parameter works correctly -* Fixed bug where file artifacts wouldn't be saved correctly in offline mode if sender runs in different location to script -* Fixed bug where DEBUG log messages were spamming to the console -* Fixed link to run dashboard printed to the console by removing `/api` -* Fixed bug where offline mode wouldn't work if no run name provided -* Fixed bug where errors would be thrown if a traceback was logged as an event when a run was already terminated -* Fixed hierarchical artifact retrieval to maintain directory structure -* Loosened Numpy requirement to >2.0.0 -* Add support for defining Simvue run defaults using `tool.simvue` in a project `pyproject.toml` file. -* Drop support for INI based configuration files. -* Retrieve all metric values if `max_points` is unspecified or set to `None`. -* Add support for PyTorch in Python 3.13 -* Create lower level API for directly interacting with the Simvue RestAPI endpoints. -* **Removes support for Python <3.10 due to dependency constraints.** -* Separates `create_alert` into specific methods `create_event_alert` etc. -* Adds additional functionality and support for offline mode. -* Support for Simvue servers `>=3`. + +- Add new example notebooks +- Update and refactor examples to work with v2.0 +- Fix bug in offline artifacts using wrong file path +- Change names of sustainability metrics +- Fix `Self` being used in typing Generators so that Simvue works with Python 3.10 in Conda +- Updated codecarbon to work with new API +- Codecarbon now works with offline mode +- Codecarbon metadata dict is now nested +- Add PID to sender lock file so it can recover from crashes +- Add accept Gzip encoding +- Fixed list of processes to add / remove from existing list of objects +- Add step to resource metrics +- Fix bug where process user alerts should not be overridden if manually set by the user +- Removed 'no config file' and 'unstaged changes' warnings from Offline mode as they do not apply +- Made `staging_check` not apply in Offline mode +- Added heartbeat functionality to Offline mode +- Moved away from `FlatDict` module for metadata collection, fixes Simvue in Jupyter notebooks +- Fixed `reconnect()` by setting `read_only` to False and added tests +- Fixed resource metrics collection to return measurement on startup and use short interval for more accurate measurements +- Fixed `set_pid` so that resource metrics are also collected for child processes of it +- Improved sender by having all cached files read at start and lock file so only one sender runs at once +- Added `name` option in `log_alert` and added tests +- Fixed client `get_alerts` and improved tests +- Removed all server config checks in Offline mode +- Fixed `add_alerts` so that it now works with both IDs and names +- Improved alert and folder deduplication methods to rely on 409 responses from server upon creation +- Added `attach_to_run` option to create alerts methods so that alerts can be created without a run attached +- Improved merging of local staging file and \_staged dict using `deepmerge` - fixes bugs with tags, alerts and metadata in offline mode +- Added `started`, `created` and `ended` timestamps to runs in offline mode +- Remove all erronous server calls in offline mode +- Fixed method to find simvue.toml config files, now just looks in cwd and home +- Added run notification option to `run.init` so that users can now get emails upon their runs completing +- Fixed artifact retrieval by run so that `category` parameter works correctly +- Fixed bug where file artifacts wouldn't be saved correctly in offline mode if sender runs in different location to script +- Fixed bug where DEBUG log messages were spamming to the console +- Fixed link to run dashboard printed to the console by removing `/api` +- Fixed bug where offline mode wouldn't work if no run name provided +- Fixed bug where errors would be thrown if a traceback was logged as an event when a run was already terminated +- Fixed hierarchical artifact retrieval to maintain directory structure +- Loosened Numpy requirement to >2.0.0 +- Add support for defining Simvue run defaults using `tool.simvue` in a project `pyproject.toml` file. +- Drop support for INI based configuration files. +- Retrieve all metric values if `max_points` is unspecified or set to `None`. +- Add support for PyTorch in Python 3.13 +- Create lower level API for directly interacting with the Simvue RestAPI endpoints. +- **Removes support for Python <3.10 due to dependency constraints.** +- Separates `create_alert` into specific methods `create_event_alert` etc. +- Adds additional functionality and support for offline mode. +- Support for Simvue servers `>=3`. ## [v1.1.4](https://github.com/simvue-io/python-api/releases/tag/v1.1.4) - 2024-12-11 -* Remove incorrect identifier reference for latest Simvue servers during reconnection. -* Fixed missing online mode selection when retrieving configuration for `Client` class. +- Remove incorrect identifier reference for latest Simvue servers during reconnection. +- Fixed missing online mode selection when retrieving configuration for `Client` class. ## [v1.1.3](https://github.com/simvue-io/python-api/releases/tag/v1.1.3) - 2024-12-09 -* Fixed bug with `requirements.txt` metadata read. -* Added Simvue server version check. -* Remove checking of server version in offline mode and add default run mode to configuration options. -* Fix offline mode class initialisation, and propagation of configuration. +- Fixed bug with `requirements.txt` metadata read. +- Added Simvue server version check. +- Remove checking of server version in offline mode and add default run mode to configuration options. +- Fix offline mode class initialisation, and propagation of configuration. ## [v1.1.2](https://github.com/simvue-io/python-api/releases/tag/v1.1.2) - 2024-11-06 -* Fix bug in offline mode directory retrieval. +- Fix bug in offline mode directory retrieval. ## [v1.1.1](https://github.com/simvue-io/python-api/releases/tag/v1.1.1) - 2024-10-22 -* Add missing `offline.cache` key to TOML config. -* Fix repetition of server URL validation for each call to configuration. +- Add missing `offline.cache` key to TOML config. +- Fix repetition of server URL validation for each call to configuration. ## [v1.1.0](https://github.com/simvue-io/python-api/releases/tag/v1.1.0) - 2024-10-21 -* Add option to specify a callback executed when an alert is triggered for a run. -* Allow retrieval of all alerts when no constraints are specified. -* Add carbon emissions statistics as optional metrics. -* Include Python and Rust environment metadata. -* Allow the disabling of heartbeat to allow runs to continue indefinitely. -* Verify Simvue server URL as early as possible. -* Indicate the source used for token and URL. -* Migrate to `simvue.toml` from `simvue.ini`, allowing more defaults to be set during runs. +- Add option to specify a callback executed when an alert is triggered for a run. +- Allow retrieval of all alerts when no constraints are specified. +- Add carbon emissions statistics as optional metrics. +- Include Python and Rust environment metadata. +- Allow the disabling of heartbeat to allow runs to continue indefinitely. +- Verify Simvue server URL as early as possible. +- Indicate the source used for token and URL. +- Migrate to `simvue.toml` from `simvue.ini`, allowing more defaults to be set during runs. ## [v1.0.6](https://github.com/simvue-io/python-api/releases/tag/v1.0.6) - 2024-10-10 -* Fix incorrect usage of `retry` when attempting connections to the server. +- Fix incorrect usage of `retry` when attempting connections to the server. ## [v1.0.5](https://github.com/simvue-io/python-api/releases/tag/v1.0.5) - 2024-10-09 -* Ensure all functionality is deactivated when mode is set to `disabled`. -* When an exception is thrown an event is sent to Simvue displaying the traceback. -* If `add_process` is used and an exception is thrown, `.err` and `.out` files are still uploaded. +- Ensure all functionality is deactivated when mode is set to `disabled`. +- When an exception is thrown an event is sent to Simvue displaying the traceback. +- If `add_process` is used and an exception is thrown, `.err` and `.out` files are still uploaded. ## [v1.0.4](https://github.com/simvue-io/python-api/releases/tag/v1.0.4) - 2024-09-24 -* Set resource metrics to be recorded by default. +- Set resource metrics to be recorded by default. ## [v1.0.3](https://github.com/simvue-io/python-api/releases/tag/v1.0.3) - 2024-09-23 -* Fix issue of hanging threads when exception raised by script using the API. +- Fix issue of hanging threads when exception raised by script using the API. ## [v1.0.2](https://github.com/simvue-io/python-api/releases/tag/v1.0.2) - 2024-08-21 -* Fix incorrect HTTP status code in `Client` when checking if object exists. -* Fix issue with `running=False` when launching a `Run` caused by incorrect system metadata being sent to the server. +- Fix incorrect HTTP status code in `Client` when checking if object exists. +- Fix issue with `running=False` when launching a `Run` caused by incorrect system metadata being sent to the server. ## [v1.0.1](https://github.com/simvue-io/python-api/releases/tag/v1.0.1) - 2024-07-16 -* Fix to `add_process` with list of strings as arguments, the executable no longer returns the string `"None"`. -* Fix callbacks and triggers for `add_process` being executed only on `Run` class termination, not on process completion. +- Fix to `add_process` with list of strings as arguments, the executable no longer returns the string `"None"`. +- Fix callbacks and triggers for `add_process` being executed only on `Run` class termination, not on process completion. ## [v1.0.0](https://github.com/simvue-io/python-api/releases/tag/v1.0.0) - 2024-06-14 -* Refactor and re-write of codebase to align with latest developments in version 2 of the Simvue server. -* Added `Executor` to Simvue runs allowing users to start shell based processes as part of a run and handle termination of these. -* Removal of obsolete functions due to server change, and renaming of functions and parameters (see [documentation](https://docs.simvue.io)). -* Added pre-request validation to both `Client` and `Run` class methods via Pydantic. -* Separation of save functionality into `save_file` and `save_object`. -* Fixed issue whereby metrics would still have to wait for the next iteration of dispatch before being sent to the server, even if the queue was not full. -* Added support for `'user'` alerts. +- Refactor and re-write of codebase to align with latest developments in version 2 of the Simvue server. +- Added `Executor` to Simvue runs allowing users to start shell based processes as part of a run and handle termination of these. +- Removal of obsolete functions due to server change, and renaming of functions and parameters (see [documentation](https://docs.simvue.io)). +- Added pre-request validation to both `Client` and `Run` class methods via Pydantic. +- Separation of save functionality into `save_file` and `save_object`. +- Fixed issue whereby metrics would still have to wait for the next iteration of dispatch before being sent to the server, even if the queue was not full. +- Added support for `'user'` alerts. ## [v0.14.3](https://github.com/simvue-io/python-api/releases/tag/v0.14.3) - 2023-06-29 -* Ensure import of the `requests` module is only done if actually used. +- Ensure import of the `requests` module is only done if actually used. ## [v0.14.0](https://github.com/simvue-io/python-api/releases/tag/v0.14.0) - 2023-04-04 -* Added a method to the `Client` class for retrieving events. +- Added a method to the `Client` class for retrieving events. ## [v0.13.3](https://github.com/simvue-io/python-api/releases/tag/v0.13.3) - 2023-04-04 -* Allow files (`input` and `code` only) to be saved for runs in the `created` state. -* Allow metadata and tags to be updated for runs in the `created` state. +- Allow files (`input` and `code` only) to be saved for runs in the `created` state. +- Allow metadata and tags to be updated for runs in the `created` state. ## [v0.13.2](https://github.com/simvue-io/python-api/releases/tag/v0.13.2) - 2023-04-04 -* Added `plot_metrics` method to the `Client` class to simplify plotting metrics. -* (Bug fix) `reconnect` works without a uuid being specified when `offline` mode isn't being used. -* (Bug fix) Restrict version of Pydantic to prevent v2 from accidentally being used. +- Added `plot_metrics` method to the `Client` class to simplify plotting metrics. +- (Bug fix) `reconnect` works without a uuid being specified when `offline` mode isn't being used. +- (Bug fix) Restrict version of Pydantic to prevent v2 from accidentally being used. ## [v0.13.1](https://github.com/simvue-io/python-api/releases/tag/v0.13.1) - 2023-03-28 -* Set `sample_by` to 0 by default (no sampling) in `get_metrics_multiple`. +- Set `sample_by` to 0 by default (no sampling) in `get_metrics_multiple`. ## [v0.13.0](https://github.com/simvue-io/python-api/releases/tag/v0.13.0) - 2023-03-28 -* Added methods to the `Client` class for retrieving metrics. -* CPU architecture and processor obtained on Apple hardware. -* Client now reports to server when files have been successfully uploaded. -* `User-Agent` header now included in HTTP requests. +- Added methods to the `Client` class for retrieving metrics. +- CPU architecture and processor obtained on Apple hardware. +- Client now reports to server when files have been successfully uploaded. +- `User-Agent` header now included in HTTP requests. ## [v0.12.0](https://github.com/simvue-io/python-api/releases/tag/v0.12.0) - 2023-03-13 -* Add methods to the `Client` class for deleting runs and folders. -* Confusing messages about `process no longer exists` or `NVML Shared Library Not Found` no longer displayed. +- Add methods to the `Client` class for deleting runs and folders. +- Confusing messages about `process no longer exists` or `NVML Shared Library Not Found` no longer displayed. ## [v0.11.4](https://github.com/simvue-io/python-api/releases/tag/v0.11.4) - 2023-03-13 -* (Bug fix) Ensure `simvue_sender` can be run when installed from PyPI. -* (Bug fix) Runs created in `offline` mode using a context manager weren't automatically closed. +- (Bug fix) Ensure `simvue_sender` can be run when installed from PyPI. +- (Bug fix) Runs created in `offline` mode using a context manager weren't automatically closed. ## [v0.11.3](https://github.com/simvue-io/python-api/releases/tag/v0.11.3) - 2023-03-07 -* Added logging messages for debugging when debug level set to `debug`. +- Added logging messages for debugging when debug level set to `debug`. ## [v0.11.2](https://github.com/simvue-io/python-api/releases/tag/v0.11.2) - 2023-03-06 -* Raise exceptions in `Client` class methods if run does not exist or artifact does not exist. -* (Bug fix) `list_artifacts` optional category restriction now works. +- Raise exceptions in `Client` class methods if run does not exist or artifact does not exist. +- (Bug fix) `list_artifacts` optional category restriction now works. ## [v0.11.1](https://github.com/simvue-io/python-api/releases/tag/v0.11.1) - 2023-03-05 -* Support different runs having different metadata in `get_runs` dataframe output. -* (Bug fix) Error message when creating a duplicate run is now more clear. -* (Bug fix) Correction to stopping the worker thread in situations where the run never started. +- Support different runs having different metadata in `get_runs` dataframe output. +- (Bug fix) Error message when creating a duplicate run is now more clear. +- (Bug fix) Correction to stopping the worker thread in situations where the run never started. ## [v0.11.0](https://github.com/simvue-io/python-api/releases/tag/v0.11.0) - 2023-03-04 -* Support optional dataframe output from `get_runs`. +- Support optional dataframe output from `get_runs`. ## [v0.10.1](https://github.com/simvue-io/python-api/releases/tag/v0.10.1) - 2023-03-03 -* The worker process now no longer gives a long delay when a run has finished (now at most ~1 second). -* The worker process ends when the `Run()` context ends or `close` is called, rather than only when the main process exits. +- The worker process now no longer gives a long delay when a run has finished (now at most ~1 second). +- The worker process ends when the `Run()` context ends or `close` is called, rather than only when the main process exits. ## [v0.10.0](https://github.com/simvue-io/python-api/releases/tag/v0.10.0) - 2023-02-07 -* The `client` class can now be used to retrieve runs. +- The `client` class can now be used to retrieve runs. ## [v0.9.1](https://github.com/simvue-io/python-api/releases/tag/v0.9.1) - 2023-01-25 -* (Bug fix) Retries in POST/PUTs to REST APIs didn't happen. -* Warn users if `allow_pickle=True` is required. +- (Bug fix) Retries in POST/PUTs to REST APIs didn't happen. +- Warn users if `allow_pickle=True` is required. ## [v0.9.0](https://github.com/simvue-io/python-api/releases/tag/v0.9.0) - 2023-01-25 -* Set status to `failed` or `terminated` if the context manager is used and there is an exception. +- Set status to `failed` or `terminated` if the context manager is used and there is an exception. ## [v0.8.0](https://github.com/simvue-io/python-api/releases/tag/v0.8.0) - 2023-01-23 -* Support NumPy arrays, PyTorch tensors, Matplotlib and Plotly plots and picklable Python objects as artifacts. -* (Bug fix) Events in offline mode didn't work. +- Support NumPy arrays, PyTorch tensors, Matplotlib and Plotly plots and picklable Python objects as artifacts. +- (Bug fix) Events in offline mode didn't work. ## [v0.7.2](https://github.com/simvue-io/python-api/releases/tag/v0.7.2) - 2023-01-08 -* Pydantic model is used for input validation. -* Support NaN, -inf and inf in metadata and metrics. +- Pydantic model is used for input validation. +- Support NaN, -inf and inf in metadata and metrics. ## [v0.7.0](https://github.com/simvue-io/python-api/releases/tag/v0.7.0) - 2022-12-05 -* Collect CPU, GPU and memory resource metrics. -* Automatically delete temporary files used in offline mode once runs have entered a terminal state. -* Warn users if their access token has expired. -* Remove dependency on the randomname module, instead handle name generation server side. +- Collect CPU, GPU and memory resource metrics. +- Automatically delete temporary files used in offline mode once runs have entered a terminal state. +- Warn users if their access token has expired. +- Remove dependency on the randomname module, instead handle name generation server side. ## [v0.6.0](https://github.com/simvue-io/python-api/releases/tag/v0.6.0) - 2022-11-07 -* `offline` and `disabled` options replaced with single `mode` flag. +- `offline` and `disabled` options replaced with single `mode` flag. ## [v0.5.0](https://github.com/simvue-io/python-api/releases/tag/v0.5.0) - 2022-11-03 -* Added option to disable all monitoring. +- Added option to disable all monitoring. ## [v0.4.0](https://github.com/simvue-io/python-api/releases/tag/v0.4.0) - 2022-11-03 -* Offline mode added, enabling tracking of simulations running on worker nodes without outgoing network access. -* Argument to `init` enabling runs to be left in the `created` state changed from `status="created"` to `running=True`. -* Improvements to error handling. +- Offline mode added, enabling tracking of simulations running on worker nodes without outgoing network access. +- Argument to `init` enabling runs to be left in the `created` state changed from `status="created"` to `running=True`. +- Improvements to error handling. ## [v0.3.0](https://github.com/simvue-io/python-api/releases/tag/v0.3.0) - 2022-10-31 -* Update `add_alert` method to support either metrics or events based alerts. +- Update `add_alert` method to support either metrics or events based alerts. ## [v0.2.0](https://github.com/simvue-io/python-api/releases/tag/v0.2.0) - 2022-10-26 -* The previous `Simvue` class has been split into `Run` and `Client`. When creating a run use the new `Run` class rather than `Simvue`. +- The previous `Simvue` class has been split into `Run` and `Client`. When creating a run use the new `Run` class rather than `Simvue`. ## [v0.1.0](https://github.com/simvue-io/client/releases/tag/v0.1.0) - 2022-10-25 -* First release. +- First release. diff --git a/poetry.lock b/poetry.lock index 610ef2d3..38084522 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1829,6 +1829,69 @@ files = [ {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, ] +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + [[package]] name = "randomname" version = "0.2.1" @@ -2145,4 +2208,4 @@ plot = ["matplotlib", "plotly"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "6f77efc12bf4a0bd08c1a107a9a45b449c3e2812737f44964769acde2937ea2c" +content-hash = "5a006e36fde3605b7ef53b51c55d4b4faa639d3b4617611650728effda1701d6" diff --git a/pyproject.toml b/pyproject.toml index e707857f..1cd4eb13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "simvue" -version = "2.1.2" +version = "2.1.3" description = "Simulation tracking and monitoring" authors = [ {name = "Simvue Development Team", email = "info@simvue.io"} @@ -55,6 +55,7 @@ dependencies = [ "deepmerge (>=2.0,<3.0)", "geocoder (>=1.38.1,<2.0.0)", "pydantic-extra-types (>=2.10.5,<3.0.0)", + "pyyaml (>=6.0.2,<7.0.0)", ] [project.urls] diff --git a/simvue/metadata.py b/simvue/metadata.py index b5dcca0a..5560ad4e 100644 --- a/simvue/metadata.py +++ b/simvue/metadata.py @@ -9,7 +9,9 @@ import contextlib import typing import json +from git import repo import toml +import yaml import logging import pathlib @@ -76,6 +78,62 @@ def git_info(repository: str) -> dict[str, typing.Any]: return {} +def _conda_env(environment_file: pathlib.Path) -> dict[str, str]: + """Parse/interpret a Conda environment file.""" + content = yaml.load(environment_file.open(), Loader=yaml.SafeLoader) + python_environment: dict[str, str] = {} + pip_dependencies: list[str] = [] + for dependency in content.get("dependencies", []): + if isinstance(dependency, dict) and dependency.get("pip"): + pip_dependencies = dependency["pip"] + break + + for dependency in pip_dependencies: + if dependency.startswith("::"): + logger.warning( + f"Skipping Conda specific channel definition '{dependency}' in Python environment metadata." + ) + elif ">=" in dependency: + module, version = dependency.split(">=") + logger.warning( + f"Ignoring '>=' constraint in Python package version, naively storing '{module}=={version}', " + "for a more accurate record use 'conda env export > environment.yml'" + ) + python_environment[module.strip()] = version.strip() + elif "~=" in dependency: + module, version = dependency.split("~=") + logger.warning( + f"Ignoring '~=' constraint in Python package version, naively storing '{module}=={version}', " + "for a more accurate record use 'conda env export > environment.yml'" + ) + python_environment[module.strip()] = version.strip() + elif dependency.startswith("-e"): + _, version = dependency.split("-e") + version = version.strip() + module = pathlib.Path(version).name + python_environment[module.strip()] = version.strip() + elif dependency.startswith("file://"): + _, version = dependency.split("file://") + module = pathlib.Path(version).name + python_environment[module.strip()] = version.strip() + elif dependency.startswith("git+"): + _, version = dependency.split("git+") + if "#egg=" in version: + repo, module = version.split("#egg=") + else: + module = version.split("/")[-1].replace(".git", "") + module = pathlib.Path(version).name + python_environment[module.strip()] = version.strip() + elif "==" not in dependency: + logger.warning( + f"Ignoring '{dependency}' in Python environment record as no version constraint specified." + ) + else: + module, version = dependency.split("==") + python_environment[module.strip()] = version.strip() + return python_environment + + def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]: """Retrieve a dictionary of Python dependencies if lock file is available""" python_meta: dict[str, dict] = {} @@ -103,6 +161,12 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]: python_meta["environment"] = { package["name"]: package["version"] for package in content } + # Handle Conda case, albeit naively given the user may or may not have used 'conda env' + # to dump their exact dependency versions + elif ( + environment_file := pathlib.Path(repository).joinpath("environment.yml") + ).exists(): + return _conda_env(environment_file) else: with contextlib.suppress((KeyError, ImportError)): from pip._internal.operations.freeze import freeze diff --git a/tests/example_data/python_conda/environment.yml b/tests/example_data/python_conda/environment.yml new file mode 100644 index 00000000..50b596be --- /dev/null +++ b/tests/example_data/python_conda/environment.yml @@ -0,0 +1,40 @@ +name: advanced_env +channels: + - conda-forge + - anaconda + - defaults +dependencies: + # Basic Conda packages with different version specifiers + - python=3.10.12 + - numpy>=1.23.5 + - pandas + - scikit-learn<1.2 + - openjdk>=11,<12 + + # Platform-specific dependencies + - libsass # Standard dependency + - vsix-installer # Standard dependency + - openblas # A package that may have platform-specific builds + + # Using a sub-channel (also called a label) + - ::my-package-from-subchannel + + # A 'pip' section for installing packages from PyPI and other sources + - pip + - pip: + # Public PyPI packages with different version specifiers + - requests==2.31.0 + - black + - jupyterlab~=4.0.0 + - numpy==2.32.2 + + # A local package from a path + - -e ./path/to/my-local-package + - file:///path/to/my-local-wheel.whl + + # A package from a Git repository + - git+https://github.com/myuser/myrepo.git#egg=myproject + +variables: + # Define environment variables + MY_ENV_VAR: "some_value" diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 5c454e14..5375e12b 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -14,7 +14,7 @@ def test_cargo_env() -> None: @pytest.mark.metadata @pytest.mark.local @pytest.mark.parametrize( - "backend", ("poetry", "uv", None) + "backend", ("poetry", "uv", "conda", None) ) def test_python_env(backend: str | None) -> None: if backend == "poetry": @@ -23,6 +23,9 @@ def test_python_env(backend: str | None) -> None: elif backend == "uv": metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_uv")) assert metadata["project"]["name"] == "example-repo" + elif backend == "conda": + metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_conda")) + assert metadata["environment"]["requests"] else: metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data")) @@ -51,4 +54,4 @@ def test_environment() -> None: assert metadata["python"]["project"]["name"] == "example-repo" assert metadata["rust"]["project"]["name"] == "example_project" assert metadata["julia"]["project"]["name"] == "Julia Demo Project" - assert metadata["javascript"]["project"]["name"] == "my-awesome-project" \ No newline at end of file + assert metadata["javascript"]["project"]["name"] == "my-awesome-project" From d3ef5bce709e7ca425db45a4a3d3e275272fd801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 8 Aug 2025 10:17:36 +0100 Subject: [PATCH 3/5] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Modify=20module=20name?= =?UTF-8?q?=20for=20Conda=20specific=20environment.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- simvue/metadata.py | 10 +++++----- tests/functional/test_run_class.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/simvue/metadata.py b/simvue/metadata.py index 5560ad4e..436691f7 100644 --- a/simvue/metadata.py +++ b/simvue/metadata.py @@ -111,19 +111,19 @@ def _conda_env(environment_file: pathlib.Path) -> dict[str, str]: _, version = dependency.split("-e") version = version.strip() module = pathlib.Path(version).name - python_environment[module.strip()] = version.strip() + python_environment[module.strip().replace("-", "_")] = version.strip() elif dependency.startswith("file://"): _, version = dependency.split("file://") - module = pathlib.Path(version).name - python_environment[module.strip()] = version.strip() + module = pathlib.Path(version).stem + python_environment[module.strip().replace("-", "_")] = version.strip() elif dependency.startswith("git+"): _, version = dependency.split("git+") if "#egg=" in version: repo, module = version.split("#egg=") + module = repo.split("/")[-1].replace(".git", "") else: module = version.split("/")[-1].replace(".git", "") - module = pathlib.Path(version).name - python_environment[module.strip()] = version.strip() + python_environment[module.strip().replace("-", "_")] = version.strip() elif "==" not in dependency: logger.warning( f"Ignoring '{dependency}' in Python environment record as no version constraint specified." diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index c36a0b12..ebe8ceb1 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -1,5 +1,6 @@ import json import logging +import toml import os import pytest import requests @@ -1315,3 +1316,28 @@ def test_reconnect_with_process() -> None: remove_runs=True, recursive=True ) + +@pytest.mark.parametrize( + "environment", ("python_conda", "python_poetry", "python_uv", "julia", "rust", "nodejs") +) +def test_run_environment_metadata(environment: str, mocker: pytest_mock.MockerFixture) -> None: + """Tests that the environment information is compatible with the server.""" + from simvue.config.user import SimvueConfiguration + from simvue.metadata import environment as env_func + _data_dir = pathlib.Path(__file__).parents[1].joinpath("example_data") + _target_dir = _data_dir + if "python" in environment: + _target_dir = _data_dir.joinpath(environment) + _config = SimvueConfiguration.fetch() + + with sv_run.Run(server_token=_config.server.token, server_url=_config.server.url) as run: + _uuid = f"{uuid.uuid4()}".split("-")[0] + run.init( + name=f"test_run_environment_metadata_{environment}", + folder=f"/simvue_unit_testing/{_uuid}", + retention_period=os.environ.get("SIMVUE_TESTING_RETENTION_PERIOD", "2 mins"), + running=False, + visibility="tenant" if os.environ.get("CI") else None, + ) + run.update_metadata(env_func(_target_dir)) + From 32f3b79400d6d630375a9b41326caaa75b1d8f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 8 Aug 2025 13:17:37 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=9A=A8=20Fix=20linter=20warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- simvue/metadata.py | 90 ++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/simvue/metadata.py b/simvue/metadata.py index 436691f7..e398f48c 100644 --- a/simvue/metadata.py +++ b/simvue/metadata.py @@ -9,7 +9,6 @@ import contextlib import typing import json -from git import repo import toml import yaml import logging @@ -78,6 +77,49 @@ def git_info(repository: str) -> dict[str, typing.Any]: return {} +def _conda_dependency_parse(dependency: str) -> tuple[str, str] | None: + """Parse a dependency definition into module-version.""" + if dependency.startswith("::"): + logger.warning( + f"Skipping Conda specific channel definition '{dependency}' in Python environment metadata." + ) + return None + elif ">=" in dependency: + module, version = dependency.split(">=") + logger.warning( + f"Ignoring '>=' constraint in Python package version, naively storing '{module}=={version}', " + "for a more accurate record use 'conda env export > environment.yml'" + ) + elif "~=" in dependency: + module, version = dependency.split("~=") + logger.warning( + f"Ignoring '~=' constraint in Python package version, naively storing '{module}=={version}', " + "for a more accurate record use 'conda env export > environment.yml'" + ) + elif dependency.startswith("-e"): + _, version = dependency.split("-e") + version = version.strip() + module = pathlib.Path(version).name + elif dependency.startswith("file://"): + _, version = dependency.split("file://") + module = pathlib.Path(version).stem + elif dependency.startswith("git+"): + _, version = dependency.split("git+") + if "#egg=" in version: + repo, module = version.split("#egg=") + module = repo.split("/")[-1].replace(".git", "") + else: + module = version.split("/")[-1].replace(".git", "") + elif "==" not in dependency: + logger.warning( + f"Ignoring '{dependency}' in Python environment record as no version constraint specified." + ) + else: + module, version = dependency.split("==") + + return module, version + + def _conda_env(environment_file: pathlib.Path) -> dict[str, str]: """Parse/interpret a Conda environment file.""" content = yaml.load(environment_file.open(), Loader=yaml.SafeLoader) @@ -89,48 +131,10 @@ def _conda_env(environment_file: pathlib.Path) -> dict[str, str]: break for dependency in pip_dependencies: - if dependency.startswith("::"): - logger.warning( - f"Skipping Conda specific channel definition '{dependency}' in Python environment metadata." - ) - elif ">=" in dependency: - module, version = dependency.split(">=") - logger.warning( - f"Ignoring '>=' constraint in Python package version, naively storing '{module}=={version}', " - "for a more accurate record use 'conda env export > environment.yml'" - ) - python_environment[module.strip()] = version.strip() - elif "~=" in dependency: - module, version = dependency.split("~=") - logger.warning( - f"Ignoring '~=' constraint in Python package version, naively storing '{module}=={version}', " - "for a more accurate record use 'conda env export > environment.yml'" - ) - python_environment[module.strip()] = version.strip() - elif dependency.startswith("-e"): - _, version = dependency.split("-e") - version = version.strip() - module = pathlib.Path(version).name - python_environment[module.strip().replace("-", "_")] = version.strip() - elif dependency.startswith("file://"): - _, version = dependency.split("file://") - module = pathlib.Path(version).stem - python_environment[module.strip().replace("-", "_")] = version.strip() - elif dependency.startswith("git+"): - _, version = dependency.split("git+") - if "#egg=" in version: - repo, module = version.split("#egg=") - module = repo.split("/")[-1].replace(".git", "") - else: - module = version.split("/")[-1].replace(".git", "") - python_environment[module.strip().replace("-", "_")] = version.strip() - elif "==" not in dependency: - logger.warning( - f"Ignoring '{dependency}' in Python environment record as no version constraint specified." - ) - else: - module, version = dependency.split("==") - python_environment[module.strip()] = version.strip() + if not (parsed := _conda_dependency_parse(dependency)): + continue + module, version = parsed + python_environment[module.strip().replace("-", "_")] = version.strip() return python_environment From a3e56407bdb4b4d902a9df60eb8a38c346b62824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 8 Aug 2025 13:23:06 +0100 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=90=9B=20Fix=20wrong=20value=20alloca?= =?UTF-8?q?tion=20in=20metadata=20dict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- simvue/metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/simvue/metadata.py b/simvue/metadata.py index e398f48c..67b52af2 100644 --- a/simvue/metadata.py +++ b/simvue/metadata.py @@ -114,6 +114,7 @@ def _conda_dependency_parse(dependency: str) -> tuple[str, str] | None: logger.warning( f"Ignoring '{dependency}' in Python environment record as no version constraint specified." ) + return None else: module, version = dependency.split("==") @@ -170,7 +171,7 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]: elif ( environment_file := pathlib.Path(repository).joinpath("environment.yml") ).exists(): - return _conda_env(environment_file) + python_meta["environment"] = _conda_env(environment_file) else: with contextlib.suppress((KeyError, ImportError)): from pip._internal.operations.freeze import freeze