From 0fa09b62ed282f0f54ab2f776bd919430c1242f6 Mon Sep 17 00:00:00 2001 From: elseml <60779710+elseml@users.noreply.github.com> Date: Tue, 20 May 2025 13:40:24 +0200 Subject: [PATCH 1/7] Disable shuffling for validation sets (#481) --- bayesflow/datasets/offline_dataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bayesflow/datasets/offline_dataset.py b/bayesflow/datasets/offline_dataset.py index 075e5135b..8fa2657c4 100644 --- a/bayesflow/datasets/offline_dataset.py +++ b/bayesflow/datasets/offline_dataset.py @@ -40,7 +40,8 @@ def __init__( num_samples : int, optional Number of samples in the dataset. If None, it will be inferred from the data. stage : str, default="training" - Current stage (e.g., "training", "validation", etc.) used by the adapter. + Current stage (e.g., "training", "validation", etc.) used by the adapter and to disable shuffling during + validation. augmentations : dict of str to Callable or Callable, optional Dictionary of augmentation functions to apply to each corresponding key in the batch or a function to apply to the entire batch (possibly adding new keys). @@ -122,7 +123,8 @@ def num_batches(self) -> int | None: return int(np.ceil(self.num_samples / self.batch_size)) def on_epoch_end(self) -> None: - self.shuffle() + if self.stage != "validation": + self.shuffle() def shuffle(self) -> None: """Shuffle the dataset in-place.""" From 2e47608d7972d8b65dbba19f18a96b2566faa18e Mon Sep 17 00:00:00 2001 From: elseml <60779710+elseml@users.noreply.github.com> Date: Tue, 20 May 2025 13:43:34 +0200 Subject: [PATCH 2/7] Fix validation set shuffling (#481) --- bayesflow/workflows/basic_workflow.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bayesflow/workflows/basic_workflow.py b/bayesflow/workflows/basic_workflow.py index c6d0fce52..052de0898 100644 --- a/bayesflow/workflows/basic_workflow.py +++ b/bayesflow/workflows/basic_workflow.py @@ -923,7 +923,9 @@ def _fit( elif isinstance(validation_data, int): raise ValueError(f"No simulator found for generating {validation_data} data sets.") - validation_data = OfflineDataset(data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter) + validation_data = OfflineDataset( + data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter, stage="validation" + ) monitor = "val_loss" else: monitor = "loss" @@ -982,6 +984,8 @@ def _on_training_finished(self): else: file_ext = self.checkpoint_name + ".keras" - logging.info(f"""Training is now finished. + logging.info( + f"""Training is now finished. You can find the trained approximator at '{self.checkpoint_filepath}/{self.checkpoint_name}.{file_ext}'. - To load it, use approximator = keras.saving.load_model(...).""") + To load it, use approximator = keras.saving.load_model(...).""" + ) From e5484d27332acd9a8a7a1e580c1316f922f61761 Mon Sep 17 00:00:00 2001 From: elseml <60779710+elseml@users.noreply.github.com> Date: Fri, 23 May 2025 11:54:37 +0200 Subject: [PATCH 3/7] Use shuffle_on_epoch_end instead of the stage arg to control shuffling --- bayesflow/datasets/offline_dataset.py | 9 ++++++--- bayesflow/workflows/basic_workflow.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bayesflow/datasets/offline_dataset.py b/bayesflow/datasets/offline_dataset.py index 8fa2657c4..d0ecdcd45 100644 --- a/bayesflow/datasets/offline_dataset.py +++ b/bayesflow/datasets/offline_dataset.py @@ -24,6 +24,7 @@ def __init__( *, stage: str = "training", augmentations: Mapping[str, Callable] | Callable = None, + shuffle_on_epoch_end: bool = True, **kwargs, ): """ @@ -40,8 +41,7 @@ def __init__( num_samples : int, optional Number of samples in the dataset. If None, it will be inferred from the data. stage : str, default="training" - Current stage (e.g., "training", "validation", etc.) used by the adapter and to disable shuffling during - validation. + Current stage (e.g., "training", "validation", etc.) used by the adapter. augmentations : dict of str to Callable or Callable, optional Dictionary of augmentation functions to apply to each corresponding key in the batch or a function to apply to the entire batch (possibly adding new keys). @@ -52,6 +52,8 @@ def __init__( Note - augmentations are applied before the adapter is called and are generally transforms that you only want to apply during training. + shuffle_on_epoch_end : bool, default=True + Whether to shuffle the dataset at the end of each epoch (use for training but not for validation sets). **kwargs Additional keyword arguments passed to the base `PyDataset`. """ @@ -70,6 +72,7 @@ def __init__( self.indices = np.arange(self.num_samples, dtype="int64") self.augmentations = augmentations + self.shuffle_on_epoch_end = shuffle_on_epoch_end self.shuffle() @@ -123,7 +126,7 @@ def num_batches(self) -> int | None: return int(np.ceil(self.num_samples / self.batch_size)) def on_epoch_end(self) -> None: - if self.stage != "validation": + if self.shuffle_on_epoch_end: self.shuffle() def shuffle(self) -> None: diff --git a/bayesflow/workflows/basic_workflow.py b/bayesflow/workflows/basic_workflow.py index 052de0898..8797b1fcf 100644 --- a/bayesflow/workflows/basic_workflow.py +++ b/bayesflow/workflows/basic_workflow.py @@ -924,7 +924,7 @@ def _fit( raise ValueError(f"No simulator found for generating {validation_data} data sets.") validation_data = OfflineDataset( - data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter, stage="validation" + data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter, shuffle_on_epoch_end=False ) monitor = "val_loss" else: From cccf75a41995425a60aa7b8c6fdaf036c6710b69 Mon Sep 17 00:00:00 2001 From: elseml <60779710+elseml@users.noreply.github.com> Date: Tue, 27 May 2025 17:21:06 +0200 Subject: [PATCH 4/7] Control all shuffling via a single shuffle_dataset argument --- bayesflow/datasets/offline_dataset.py | 15 ++++++++------- bayesflow/workflows/basic_workflow.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bayesflow/datasets/offline_dataset.py b/bayesflow/datasets/offline_dataset.py index d0ecdcd45..da83066a9 100644 --- a/bayesflow/datasets/offline_dataset.py +++ b/bayesflow/datasets/offline_dataset.py @@ -24,7 +24,7 @@ def __init__( *, stage: str = "training", augmentations: Mapping[str, Callable] | Callable = None, - shuffle_on_epoch_end: bool = True, + shuffle_dataset: bool = True, **kwargs, ): """ @@ -52,8 +52,9 @@ def __init__( Note - augmentations are applied before the adapter is called and are generally transforms that you only want to apply during training. - shuffle_on_epoch_end : bool, default=True - Whether to shuffle the dataset at the end of each epoch (use for training but not for validation sets). + shuffle_dataset : bool, default=True + Whether to shuffle the dataset at initialization and at the end of each epoch. Should be set to `False` + for validation and test datasets to ensure consistent ordering of data. **kwargs Additional keyword arguments passed to the base `PyDataset`. """ @@ -72,9 +73,9 @@ def __init__( self.indices = np.arange(self.num_samples, dtype="int64") self.augmentations = augmentations - self.shuffle_on_epoch_end = shuffle_on_epoch_end - - self.shuffle() + self.shuffle_dataset = shuffle_dataset + if self.shuffle_dataset: + self.shuffle() def __getitem__(self, item: int) -> dict[str, np.ndarray]: """ @@ -126,7 +127,7 @@ def num_batches(self) -> int | None: return int(np.ceil(self.num_samples / self.batch_size)) def on_epoch_end(self) -> None: - if self.shuffle_on_epoch_end: + if self.shuffle_dataset: self.shuffle() def shuffle(self) -> None: diff --git a/bayesflow/workflows/basic_workflow.py b/bayesflow/workflows/basic_workflow.py index 8797b1fcf..8992cd3bf 100644 --- a/bayesflow/workflows/basic_workflow.py +++ b/bayesflow/workflows/basic_workflow.py @@ -924,7 +924,7 @@ def _fit( raise ValueError(f"No simulator found for generating {validation_data} data sets.") validation_data = OfflineDataset( - data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter, shuffle_on_epoch_end=False + data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter, shuffle_dataset=False ) monitor = "val_loss" else: From cfbd6c990895adcb1a37d316186e40059c40ae92 Mon Sep 17 00:00:00 2001 From: elseml <60779710+elseml@users.noreply.github.com> Date: Tue, 27 May 2025 17:23:13 +0200 Subject: [PATCH 5/7] Add shuffle_dataset argument --- bayesflow/datasets/disk_dataset.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bayesflow/datasets/disk_dataset.py b/bayesflow/datasets/disk_dataset.py index f94200dc8..7f26a0341 100644 --- a/bayesflow/datasets/disk_dataset.py +++ b/bayesflow/datasets/disk_dataset.py @@ -37,6 +37,7 @@ def __init__( adapter: Adapter | None, stage: str = "training", augmentations: Mapping[str, Callable] | Callable = None, + shuffle_dataset: bool = True, **kwargs, ): """ @@ -67,6 +68,9 @@ def __init__( Note - augmentations are applied before the adapter is called and are generally transforms that you only want to apply during training. + shuffle_dataset : bool, default=True + Whether to shuffle the dataset at initialization and at the end of each epoch. Should be set to `False` + for validation and test datasets to ensure consistent ordering of data. **kwargs Additional keyword arguments passed to the base `PyDataset`. """ @@ -79,8 +83,9 @@ def __init__( self.stage = stage self.augmentations = augmentations - - self.shuffle() + self.shuffle_dataset = shuffle_dataset + if self.shuffle_dataset: + self.shuffle() def __getitem__(self, item) -> dict[str, np.ndarray]: if not 0 <= item < self.num_batches: @@ -108,7 +113,8 @@ def __getitem__(self, item) -> dict[str, np.ndarray]: return batch def on_epoch_end(self): - self.shuffle() + if self.shuffle_dataset: + self.shuffle() @property def num_batches(self): From 6a8e68adb2a9ef04177ae55ab31349fbaf7fe1ed Mon Sep 17 00:00:00 2001 From: elseml <60779710+elseml@users.noreply.github.com> Date: Tue, 27 May 2025 17:36:42 +0200 Subject: [PATCH 6/7] Disable validation data shuffling in example notebooks --- .../Continuous_Consistency_Model_Playground.ipynb | 6 +++--- examples/experimental/Hyperparameter_Optimization.ipynb | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/experimental/Continuous_Consistency_Model_Playground.ipynb b/examples/experimental/Continuous_Consistency_Model_Playground.ipynb index 75446639e..843a414f1 100644 --- a/examples/experimental/Continuous_Consistency_Model_Playground.ipynb +++ b/examples/experimental/Continuous_Consistency_Model_Playground.ipynb @@ -211,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "51045bbed88cb5c2", "metadata": { "ExecuteTime": { @@ -226,7 +226,7 @@ ")\n", "\n", "validation_dataset = bf.datasets.OfflineDataset(\n", - " data=validation_samples, batch_size=batch_size, adapter=adapter\n", + " data=validation_samples, batch_size=batch_size, adapter=adapter, shuffle_dataset=False\n", ")" ] }, @@ -634,7 +634,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "bf2", "language": "python", "name": "python3" }, diff --git a/examples/experimental/Hyperparameter_Optimization.ipynb b/examples/experimental/Hyperparameter_Optimization.ipynb index 586f67b1f..496948cbf 100644 --- a/examples/experimental/Hyperparameter_Optimization.ipynb +++ b/examples/experimental/Hyperparameter_Optimization.ipynb @@ -67,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "aa529752", "metadata": {}, "outputs": [], @@ -98,7 +98,8 @@ "val_dataset = bf.datasets.OfflineDataset(\n", " data=val_samples, \n", " batch_size=batch_size, \n", - " adapter=adapter\n", + " adapter=adapter,\n", + " shuffle_dataset=False\n", ")" ] }, From 3c76da0dd6c3f7a0867a74f2891a176bb679e4bf Mon Sep 17 00:00:00 2001 From: Valentin Pratz Date: Sun, 1 Jun 2025 15:28:13 +0000 Subject: [PATCH 7/7] Limit changes to introduction of shuffle argument --- bayesflow/datasets/disk_dataset.py | 13 ++++++------- bayesflow/datasets/offline_dataset.py | 13 ++++++------- bayesflow/workflows/basic_workflow.py | 10 +++------- .../Continuous_Consistency_Model_Playground.ipynb | 6 +++--- .../experimental/Hyperparameter_Optimization.ipynb | 5 ++--- 5 files changed, 20 insertions(+), 27 deletions(-) diff --git a/bayesflow/datasets/disk_dataset.py b/bayesflow/datasets/disk_dataset.py index 7f26a0341..d776bfe82 100644 --- a/bayesflow/datasets/disk_dataset.py +++ b/bayesflow/datasets/disk_dataset.py @@ -37,7 +37,7 @@ def __init__( adapter: Adapter | None, stage: str = "training", augmentations: Mapping[str, Callable] | Callable = None, - shuffle_dataset: bool = True, + shuffle: bool = True, **kwargs, ): """ @@ -68,9 +68,8 @@ def __init__( Note - augmentations are applied before the adapter is called and are generally transforms that you only want to apply during training. - shuffle_dataset : bool, default=True - Whether to shuffle the dataset at initialization and at the end of each epoch. Should be set to `False` - for validation and test datasets to ensure consistent ordering of data. + shuffle : bool, optional + Whether to shuffle the dataset at initialization and at the end of each epoch. Default is True. **kwargs Additional keyword arguments passed to the base `PyDataset`. """ @@ -83,8 +82,8 @@ def __init__( self.stage = stage self.augmentations = augmentations - self.shuffle_dataset = shuffle_dataset - if self.shuffle_dataset: + self._shuffle = shuffle + if self._shuffle: self.shuffle() def __getitem__(self, item) -> dict[str, np.ndarray]: @@ -113,7 +112,7 @@ def __getitem__(self, item) -> dict[str, np.ndarray]: return batch def on_epoch_end(self): - if self.shuffle_dataset: + if self._shuffle: self.shuffle() @property diff --git a/bayesflow/datasets/offline_dataset.py b/bayesflow/datasets/offline_dataset.py index da83066a9..3b91c5f22 100644 --- a/bayesflow/datasets/offline_dataset.py +++ b/bayesflow/datasets/offline_dataset.py @@ -24,7 +24,7 @@ def __init__( *, stage: str = "training", augmentations: Mapping[str, Callable] | Callable = None, - shuffle_dataset: bool = True, + shuffle: bool = True, **kwargs, ): """ @@ -52,9 +52,8 @@ def __init__( Note - augmentations are applied before the adapter is called and are generally transforms that you only want to apply during training. - shuffle_dataset : bool, default=True - Whether to shuffle the dataset at initialization and at the end of each epoch. Should be set to `False` - for validation and test datasets to ensure consistent ordering of data. + shuffle : bool, optional + Whether to shuffle the dataset at initialization and at the end of each epoch. Default is True. **kwargs Additional keyword arguments passed to the base `PyDataset`. """ @@ -73,8 +72,8 @@ def __init__( self.indices = np.arange(self.num_samples, dtype="int64") self.augmentations = augmentations - self.shuffle_dataset = shuffle_dataset - if self.shuffle_dataset: + self._shuffle = shuffle + if self._shuffle: self.shuffle() def __getitem__(self, item: int) -> dict[str, np.ndarray]: @@ -127,7 +126,7 @@ def num_batches(self) -> int | None: return int(np.ceil(self.num_samples / self.batch_size)) def on_epoch_end(self) -> None: - if self.shuffle_dataset: + if self._shuffle: self.shuffle() def shuffle(self) -> None: diff --git a/bayesflow/workflows/basic_workflow.py b/bayesflow/workflows/basic_workflow.py index 8992cd3bf..c6d0fce52 100644 --- a/bayesflow/workflows/basic_workflow.py +++ b/bayesflow/workflows/basic_workflow.py @@ -923,9 +923,7 @@ def _fit( elif isinstance(validation_data, int): raise ValueError(f"No simulator found for generating {validation_data} data sets.") - validation_data = OfflineDataset( - data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter, shuffle_dataset=False - ) + validation_data = OfflineDataset(data=validation_data, batch_size=dataset.batch_size, adapter=self.adapter) monitor = "val_loss" else: monitor = "loss" @@ -984,8 +982,6 @@ def _on_training_finished(self): else: file_ext = self.checkpoint_name + ".keras" - logging.info( - f"""Training is now finished. + logging.info(f"""Training is now finished. You can find the trained approximator at '{self.checkpoint_filepath}/{self.checkpoint_name}.{file_ext}'. - To load it, use approximator = keras.saving.load_model(...).""" - ) + To load it, use approximator = keras.saving.load_model(...).""") diff --git a/examples/experimental/Continuous_Consistency_Model_Playground.ipynb b/examples/experimental/Continuous_Consistency_Model_Playground.ipynb index 843a414f1..75446639e 100644 --- a/examples/experimental/Continuous_Consistency_Model_Playground.ipynb +++ b/examples/experimental/Continuous_Consistency_Model_Playground.ipynb @@ -211,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "51045bbed88cb5c2", "metadata": { "ExecuteTime": { @@ -226,7 +226,7 @@ ")\n", "\n", "validation_dataset = bf.datasets.OfflineDataset(\n", - " data=validation_samples, batch_size=batch_size, adapter=adapter, shuffle_dataset=False\n", + " data=validation_samples, batch_size=batch_size, adapter=adapter\n", ")" ] }, @@ -634,7 +634,7 @@ ], "metadata": { "kernelspec": { - "display_name": "bf2", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, diff --git a/examples/experimental/Hyperparameter_Optimization.ipynb b/examples/experimental/Hyperparameter_Optimization.ipynb index 496948cbf..586f67b1f 100644 --- a/examples/experimental/Hyperparameter_Optimization.ipynb +++ b/examples/experimental/Hyperparameter_Optimization.ipynb @@ -67,7 +67,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "aa529752", "metadata": {}, "outputs": [], @@ -98,8 +98,7 @@ "val_dataset = bf.datasets.OfflineDataset(\n", " data=val_samples, \n", " batch_size=batch_size, \n", - " adapter=adapter,\n", - " shuffle_dataset=False\n", + " adapter=adapter\n", ")" ] },