From 23cbba88c3f44dbccabf906a651f2b992481ef9d Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Fri, 13 Feb 2026 09:05:38 -0500 Subject: [PATCH 01/30] Removed the argument "batch_size" from the trainers. Changed default hyperparameters in the models. Added demo for profile reconstruction. Added script for dataset standardization (has to be run once before model training to store normalization coefficients). --- scripts/fast_time_series_reconstruction.py | 19 +-- scripts/profile_reconstruction.py | 83 +++++++++++ scripts/run_demo.py | 137 ++++++------------ scripts/run_demo_2.py | 15 +- scripts/standardize_dataset.py | 24 +++ .../modality/fast_time_series_baseline.py | 15 +- .../trainer/trainer.py | 4 - 7 files changed, 179 insertions(+), 118 deletions(-) create mode 100644 scripts/profile_reconstruction.py create mode 100644 scripts/standardize_dataset.py diff --git a/scripts/fast_time_series_reconstruction.py b/scripts/fast_time_series_reconstruction.py index cbf70c7..06eb602 100644 --- a/scripts/fast_time_series_reconstruction.py +++ b/scripts/fast_time_series_reconstruction.py @@ -14,9 +14,11 @@ class DummyModel(torch.nn.Module): def __init__(self): super(DummyModel, self).__init__() self.encoder = TimeSeriesEncoder( - n_channels=6, input_length=5000, d_model=512, n_output_tokens=100) + kernel_size=11, n_channels=8, input_length=5000, d_model=512, + n_output_tokens=100) self.decoder = TimeSeriesDecoder( - n_channels=6, input_length=5000, d_model=512, n_input_tokens=100) + kernel_size=11, n_channels=8, input_length=5000, d_model=512, + n_input_tokens=100) def forward(self, x): x_encoded = self.encoder(x) @@ -55,8 +57,8 @@ def worker_init_fn(worker_id): TokamakH5Dataset( hdf5_path=str(f), preprocessing_stats=stats, - input_signals=["d_alpha", ], - target_signals=["d_alpha", ], + input_signals=["pin", ], + target_signals=["pin", ], prediction_mode=False, ) for f in hdf5_files @@ -66,16 +68,15 @@ def worker_init_fn(worker_id): dataloader = DataLoader( concatenated_dataset, - batch_size=2, + batch_size=8, shuffle=False, collate_fn=collate_fn, worker_init_fn=worker_init_fn ) -optimizer = optim.AdamW(model.parameters(), lr=0.001) +optimizer = optim.AdamW(model.parameters(), lr=0.005) loss_fn = nn.MSELoss() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) -trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=10, - batch_size=2) -trainer.train(dataloader, modality_key="d_alpha") +trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50) +trainer.train(dataloader, val_dataloader=dataloader, modality_key="pin") diff --git a/scripts/profile_reconstruction.py b/scripts/profile_reconstruction.py new file mode 100644 index 0000000..a0e12c9 --- /dev/null +++ b/scripts/profile_reconstruction.py @@ -0,0 +1,83 @@ +from pathlib import Path +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.models.modality.profile_baseline import ( + SpatialProfileEncoder, SpatialProfileDecoder) +from tokamak_foundation_model.trainer.trainer import UnimodalTrainer + + +class DummyModel(torch.nn.Module): + def __init__(self): + super(DummyModel, self).__init__() + self.encoder = SpatialProfileEncoder( + kernel_size=3, n_spatial_points=44, n_time_points=50, d_model=512, + n_output_tokens=100) + self.decoder = SpatialProfileDecoder( + kernel_size=3, n_spatial_points=44, n_time_points=50, d_model=512, + n_input_tokens=100) + + def forward(self, x): + x_encoded = self.encoder(x) + return self.decoder(x_encoded) + + +def worker_init_fn(worker_id): + """Each worker needs to open its own file handle.""" + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None: + dataset = worker_info.dataset + # Force re-open file for this worker + if hasattr(dataset, 'datasets'): # ConcatDataset + for ds in dataset.datasets: + ds.h5_file = None + ds._open_hdf5() + else: + dataset.h5_file = None + dataset._open_hdf5() + + +model = DummyModel() + + +hdf5_files = sorted( + Path( + "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/tokamak_package/" + ).glob("*_processed.h5") +) +stats = torch.load( + "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/" + "tokamak_package/preprocessing_stats.pt" +) + +datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=["ts_core_density", ], + target_signals=["ts_core_density", ], + prediction_mode=False, + ) + for f in hdf5_files +] + +concatenated_dataset = ConcatDataset(datasets_processed) + +dataloader = DataLoader( + concatenated_dataset, + batch_size=8, + shuffle=False, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn + ) + +optimizer = optim.AdamW(model.parameters(), lr=0.005) +loss_fn = nn.L1Loss() # Be careful +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = model.to(device) +trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50) +trainer.train(dataloader, val_dataloader=dataloader, modality_key="ts_core_density") + diff --git a/scripts/run_demo.py b/scripts/run_demo.py index d129aa5..d886dc9 100644 --- a/scripts/run_demo.py +++ b/scripts/run_demo.py @@ -1,14 +1,8 @@ -import numpy as np from pathlib import Path import torch -import torch.nn as nn -import torch.optim as optim -from torch.utils.data import DataLoader, ConcatDataset +from torch.utils.data import ConcatDataset -from tokamak_foundation_model.data.data_loader import ( - TokamakH5Dataset, collate_fn, collate_fn_prediction, compute_preprocessing_stats) -from tokamak_foundation_model.models.dummy_model import MultiModalTokamakModel, MultiModalPredictionModel -from tokamak_foundation_model.trainer.trainer import Trainer +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset def worker_init_fn(worker_id): @@ -25,91 +19,46 @@ def worker_init_fn(worker_id): dataset.h5_file = None dataset._open_hdf5() -print("Initializing and demonstrating custom DataLoader with updated TokamakH5Dataset") -# Use glob to find all generated HDF5 files -hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data").glob("*_processed.h5") - ) - -# Create TokamakH5Dataset instances for each HDF5 file -# datasets = [TokamakH5Dataset(hdf5_path=str(f)) for f in hdf5_files] -# stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') -stats = torch.load('data/preprocessing_stats.pt') - -# All signals the model expects as inputs -all_input_signals = [ - "mhr", "ece", "co2", # spectrograms - "gas", "ech", "pin", "tin", # actuators - "d_alpha", "mse", "ts_core_density", # diagnostics - "bolo", "irtv", "tangtv", # videos - "text", # metadata -] - -datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=all_input_signals, - ) for f in hdf5_files] -# Concatenate the datasets -concatenated_dataset = ConcatDataset(datasets_processed) - -print(f"Initialized ConcatDataset with {len(concatenated_dataset)} samples.") - -# Initialize DataLoader -dataloader = DataLoader( - concatenated_dataset, - batch_size=2, - shuffle=False, - collate_fn=collate_fn_prediction, - worker_init_fn=worker_init_fn +def data_loading_demo(): + print("Initializing and demonstrating custom DataLoader with updated TokamakH5Dataset") + # Use glob to find all generated HDF5 files + hdf5_files = sorted( + Path("C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/" + "tokamak_package/").glob("*_processed.h5") ) - -# Get and print the first batch from DataLoader to verify functionality -batch = next(iter(dataloader)) # Get the first batch to verify functionality - -# --- 3. Initialize and Demonstrate Dummy PyTorch Model with text input --- -print("\n--- 3. Initializing and demonstrating Dummy PyTorch Model with text input ---") -model = MultiModalPredictionModel() -print("MultiModalPredictionModel structure:") -print(model) - -model.eval() -with torch.no_grad(): - # The batch now includes 'text' data - output = model(batch) -print(f"Model output type: {type(output)}, shape: {output.shape}") - -# # --- 4. Initialize and Demonstrate Extensible PyTorch Trainer --- -# print("\n--- 4. Initializing and demonstrating Extensible PyTorch Trainer ---") -# optimizer = optim.Adam(model.parameters(), lr=0.001) -# loss_fn = nn.MSELoss() # Dummy loss for regression -# device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -# model.to(device) -# print(f"Using device: {device}") - -# trainer = Trainer( -# model=model, -# optimizer=optimizer, -# loss_fn=loss_fn, -# device=device, -# epochs=10, # Only 1 epoch for demonstration -# batch_size=2, -# checkpoint_path="dummy_trainer_checkpoint.pth" -# ) -# print("Trainer class initialized.") - -# print("Running dummy training epoch...") -# # Ensure the model is in training mode before calling _train_epoch -# model.train() -# train_metrics = trainer.train(dataloader) # Corrected method call -# print(f" Finished dummy training epoch. Metrics: {train_metrics}") - -# print("Running dummy validation epoch...") -# # Ensure the model is in evaluation mode before calling _validate_epoch -# model.eval() -# val_metrics = trainer._validate_epoch(dataloader) # Corrected method call -# print(f" Finished dummy validation epoch. Metrics: {val_metrics}") - -# print("\nDemonstration complete!") + stats = torch.load( + "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/" + "tokamak_package/preprocessing_stats.pt" + ) + all_input_signals = [ + "mhr", + "ece", + "co2", # spectrograms + "gas", + "ech", + "pin", + "tin", # actuators + "d_alpha", + "mse", + "ts_core_density", # diagnostics + "bolo", + "irtv", + "tangtv", # videos + "text", # metadata + ] + + datasets_processed = [TokamakH5Dataset(hdf5_path=str(f), preprocessing_stats=stats, + input_signals=all_input_signals, + target_signals=all_input_signals, + prediction_mode=False) for f in hdf5_files] + + concatenated_dataset = ConcatDataset(datasets_processed) + + + # Get and print the first batch from DataLoader to verify functionality + for k in range(len(concatenated_dataset)): + concatenated_dataset.__getitem__(k) + +if __name__ == "__main__": + data_loading_demo() diff --git a/scripts/run_demo_2.py b/scripts/run_demo_2.py index 2393709..ff00697 100644 --- a/scripts/run_demo_2.py +++ b/scripts/run_demo_2.py @@ -7,9 +7,9 @@ from torchinfo import summary from tokamak_foundation_model.data.data_loader import ( - TokamakH5Dataset, collate_fn, collate_fn_prediction, compute_preprocessing_stats) + TokamakH5Dataset, collate_fn_prediction, compute_preprocessing_stats) from tokamak_foundation_model.models.dummy_model_2 import MultiModalTokamakModel, MultiModalPredictionModel -from tokamak_foundation_model.trainer.trainer import Trainer +from tokamak_foundation_model.trainer.trainer import MultimodalTrainer def worker_init_fn(worker_id): @@ -29,13 +29,16 @@ def worker_init_fn(worker_id): print("Initializing and demonstrating custom DataLoader with updated TokamakH5Dataset") # Use glob to find all generated HDF5 files hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data").glob("*_processed.h5") - ) + Path( + r"C:\Users\admin\PycharmProjects\nstx\foundation_model_notes\tokamak_package" + ).glob("*_processed.h5") +) # Create TokamakH5Dataset instances for each HDF5 file # datasets = [TokamakH5Dataset(hdf5_path=str(f)) for f in hdf5_files] # stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') -stats = torch.load('data/preprocessing_stats.pt') +stats = torch.load(r'C:\Users\admin\PycharmProjects\nstx\foundation_model_notes' + r'\tokamak_package/preprocessing_stats.pt') # All signals the model expects as inputs all_input_signals = [ @@ -91,7 +94,7 @@ def worker_init_fn(worker_id): model.to(device) print(f"Using device: {device}") -trainer = Trainer( +trainer = MultimodalTrainer( model=model, optimizer=optimizer, loss_fn=loss_fn, diff --git a/scripts/standardize_dataset.py b/scripts/standardize_dataset.py new file mode 100644 index 0000000..61a246b --- /dev/null +++ b/scripts/standardize_dataset.py @@ -0,0 +1,24 @@ +from pathlib import Path +from tokamak_foundation_model.data.data_loader import ( + TokamakH5Dataset, compute_preprocessing_stats) + +hdf5_files = sorted( + Path( + "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/tokamak_package/" + ).glob("*_processed.h5") +) +all_input_signals = [ + "mhr", "ece", "co2", # spectrograms + "gas", "ech", "pin", "tin", # actuators + "d_alpha", "mse", "ts_core_density", # diagnostics + "bolo", "irtv", "tangtv", # videos + "text", # metadata +] + +datasets = [ + TokamakH5Dataset( + hdf5_path=str(f), + input_signals=all_input_signals, + target_signals=all_input_signals, + ) for f in hdf5_files] +stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') diff --git a/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py b/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py index b9d2f5e..f905716 100644 --- a/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py +++ b/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py @@ -99,7 +99,7 @@ def __init__( d_model: int = 512, n_output_tokens: int = 100, n_conv_layers: int = 4, - kernel_size: int = 15, + kernel_size: int = 3, verbose: bool = False ): super().__init__() @@ -134,6 +134,10 @@ def __init__( for i in range(n_conv_layers) ]) + self.norms = nn.ModuleList([ + nn.InstanceNorm1d(self.channels[i + 1]) for i in range(n_conv_layers) + ]) + self.adaptive_pool = nn.AdaptiveAvgPool1d(n_output_tokens) self.activation = nn.GELU() self.norm = nn.LayerNorm(d_model) @@ -159,12 +163,13 @@ def forward(self, x): torch.Tensor Encoded tokens of shape [batch, n_output_tokens, d_model] """ - for conv in self.conv_layers: - x = self.activation(conv(x)) # [B, channels[i+1], T'] + for conv, norm in zip(self.conv_layers, self.norms): + x = conv(x) # [B, channels[i+1], T'] + x = norm(x) + x = self.activation(x) x = self.adaptive_pool(x) # [B, d_model, n_output_tokens] x = x.transpose(1, 2) # [B, n_output_tokens, d_model] - x = self.norm(x) return x @@ -211,7 +216,7 @@ def __init__( d_model: int = 512, n_input_tokens: int = 100, n_deconv_layers: int = 4, - kernel_size: int = 15, + kernel_size: int = 3, verbose: bool = False ): super().__init__() diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index bed687e..dd01901 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -12,14 +12,12 @@ def __init__(self, loss_fn: nn.Module, device: torch.device, epochs: int, - batch_size: int, checkpoint_path: str = "checkpoint.pth"): self.model = model self.optimizer = optimizer self.loss_fn = loss_fn self.device = device self.epochs = epochs - self.batch_size = batch_size self.checkpoint_path = checkpoint_path def _train_epoch(self, dataloader: DataLoader): @@ -90,14 +88,12 @@ def __init__(self, loss_fn: nn.Module, device: torch.device, epochs: int, - batch_size: int, checkpoint_path: str = "checkpoint.pth"): self.model = model self.optimizer = optimizer self.loss_fn = loss_fn self.device = device self.epochs = epochs - self.batch_size = batch_size self.checkpoint_path = checkpoint_path def _train_epoch(self, dataloader: DataLoader, modality_key: str): From 030631b377321b3b333b737973a0e9fc9cb4bce5 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:44:31 -0500 Subject: [PATCH 02/30] Bugfix in the dataset class. When iterating over movie configurations, the wrong configuration was used to find the correct signal name. Also, removed warning for duplicated tensor conversion. --- scripts/fast_time_series_reconstruction.py | 7 +-- scripts/profile_reconstruction.py | 7 +-- .../data/data_loader.py | 26 +++++++--- .../modality/fast_time_series_baseline.py | 51 +++++++++++++++++++ 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/scripts/fast_time_series_reconstruction.py b/scripts/fast_time_series_reconstruction.py index 06eb602..e0dd2d4 100644 --- a/scripts/fast_time_series_reconstruction.py +++ b/scripts/fast_time_series_reconstruction.py @@ -44,13 +44,10 @@ def worker_init_fn(worker_id): hdf5_files = sorted( - Path( - "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/tokamak_package/" - ).glob("*_processed.h5") + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") ) stats = torch.load( - "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/" - "tokamak_package/preprocessing_stats.pt" + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") ) datasets_processed = [ diff --git a/scripts/profile_reconstruction.py b/scripts/profile_reconstruction.py index a0e12c9..6377309 100644 --- a/scripts/profile_reconstruction.py +++ b/scripts/profile_reconstruction.py @@ -44,13 +44,10 @@ def worker_init_fn(worker_id): hdf5_files = sorted( - Path( - "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/tokamak_package/" - ).glob("*_processed.h5") + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") ) stats = torch.load( - "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/" - "tokamak_package/preprocessing_stats.pt" + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") ) datasets_processed = [ diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index 7c9ce3e..ebb4583 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -305,8 +305,10 @@ def _apply_preprocessing( return tensor # Convert to tensor and reshape for broadcasting - mean = torch.tensor(config.mean, dtype=tensor.dtype, device=tensor.device) - std = torch.tensor(config.std, dtype=tensor.dtype, device=tensor.device) + mean = torch.as_tensor( + config.mean, dtype=tensor.dtype, device=tensor.device) + std = torch.as_tensor( + config.std, dtype=tensor.dtype, device=tensor.device) if reshape_dims is not None: mean = mean.reshape(reshape_dims) @@ -337,8 +339,10 @@ def _apply_preprocessing( return tensor_log # Convert to tensor and reshape for broadcasting - mean = torch.tensor(config.mean, dtype=tensor.dtype, device=tensor.device) - std = torch.tensor(config.std, dtype=tensor.dtype, device=tensor.device) + mean = torch.as_tensor( + config.mean, dtype=tensor.dtype, device=tensor.device) + std = torch.as_tensor( + config.std, dtype=tensor.dtype, device=tensor.device) if reshape_dims is not None: mean = mean.reshape(reshape_dims) @@ -630,8 +634,10 @@ def _getitem_standard(self, idx): # Load and process movies all_movies = {} for movie_config in self.MOVIE_CONFIGS: - if config.name in self.input_signals: - raw_movie = self._load_movie_raw(self.h5_file, movie_config, t_start, t_end) + if movie_config.name in self.input_signals: + raw_movie = self._load_movie_raw( + self.h5_file, movie_config, t_start, t_end + ) all_movies[movie_config.name] = raw_movie # Load metadata @@ -639,7 +645,7 @@ def _getitem_standard(self, idx): all_metadata = self._load_metadata(self.h5_file) else: all_metadata = {} - + return {**all_signals, **all_movies, **all_metadata} def _getitem_prediction(self, idx): @@ -648,15 +654,21 @@ def _getitem_prediction(self, idx): t_start = idx * self.chunk_duration_s t_end = t_start + self.chunk_duration_s + self.prediction_horizon_s + signals_to_load = set(self.input_signals) | set(self.target_signals) + # Load and process all signals with extended window all_signals = {} for config in self.SIGNAL_CONFIGS: + if config.name not in signals_to_load: + continue raw_data = self._load_signal_raw(self.h5_file, config, t_start, t_end) all_signals[config.name] = self._process_signal(raw_data, config) # Load and process movies all_movies = {} for movie_config in self.MOVIE_CONFIGS: + if movie_config.name not in signals_to_load: + continue # Load raw movie data raw_movie = self._load_movie_raw(self.h5_file, movie_config, t_start, t_end) all_movies[movie_config.name] = raw_movie diff --git a/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py b/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py index f905716..2c4fc34 100644 --- a/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py +++ b/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py @@ -285,6 +285,57 @@ def forward(self, x): return x +class TimeSeriesAutoencoder(nn.Module): + """Combines TimeSeriesEncoder and TimeSeriesDecoder into an autoencoder model.""" + + def __init__( + self, + n_channels: int = 6, + input_length: int = 5000, + d_model: int = 512, + n_tokens: int = 100, + n_layers: int = 4, + kernel_size: int = 3, + verbose: bool = False + ): + super().__init__() + self.encoder = TimeSeriesEncoder( + n_channels=n_channels, + input_length=input_length, + d_model=d_model, + n_output_tokens=n_tokens, + n_conv_layers=n_layers, + kernel_size=kernel_size, + verbose=verbose + ) + self.decoder = TimeSeriesDecoder( + n_channels=n_channels, + input_length=input_length, + d_model=d_model, + n_input_tokens=n_tokens, + n_deconv_layers=n_layers, + kernel_size=kernel_size, + verbose=verbose + ) + + def forward(self, x): + """ + Forward pass through the autoencoder. + + Parameters + ---------- + x : torch.Tensor + Input time-series of shape [batch, n_channels, input_length] + + Returns + ------- + torch.Tensor + Reconstructed time-series of shape [batch, n_channels, input_length] + """ + tokens = self.encoder(x) + recon = self.decoder(tokens) + return recon + class FastTimeSeriesEncoder(ModalityEncoder): From f07f7671e5a1d29e17fcfee59a728e1dca0eeae1 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:49:40 -0500 Subject: [PATCH 03/30] Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. --- scripts/video_reconstruction.py | 64 +++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 scripts/video_reconstruction.py diff --git a/scripts/video_reconstruction.py b/scripts/video_reconstruction.py new file mode 100644 index 0000000..8155555 --- /dev/null +++ b/scripts/video_reconstruction.py @@ -0,0 +1,64 @@ +from pathlib import Path +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.models.modality.video_baseline import ( + VideoEncoder, VideoDecoder, VideoAutoEncoder) +from tokamak_foundation_model.trainer.trainer import UnimodalTrainer + + +def worker_init_fn(worker_id): + """Each worker needs to open its own file handle.""" + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None: + dataset = worker_info.dataset + # Force re-open file for this worker + if hasattr(dataset, 'datasets'): # ConcatDataset + for ds in dataset.datasets: + ds.h5_file = None + ds._open_hdf5() + else: + dataset.h5_file = None + dataset._open_hdf5() + + +model = VideoAutoEncoder(n_tokens=100) + + +hdf5_files = sorted( + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") +) +stats = torch.load( + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") +) + +datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=["bolo", ], + target_signals=["bolo", ], + prediction_mode=False, + ) + for f in hdf5_files +] + +concatenated_dataset = ConcatDataset(datasets_processed) + +dataloader = DataLoader( + concatenated_dataset, + batch_size=2, + shuffle=False, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn + ) + +optimizer = optim.AdamW(model.parameters(), lr=0.001) +loss_fn = nn.MSELoss() +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +model = model.to(device) +trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=10) +trainer.train(dataloader, modality_key="bolo") From 4ba4756e54abc9204fd684137167144ce69a80bf Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:51:12 -0500 Subject: [PATCH 04/30] Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. --- .../models/modality/video_baseline.py | 488 ++++++++---------- 1 file changed, 214 insertions(+), 274 deletions(-) diff --git a/src/tokamak_foundation_model/models/modality/video_baseline.py b/src/tokamak_foundation_model/models/modality/video_baseline.py index 2178b8f..df21265 100644 --- a/src/tokamak_foundation_model/models/modality/video_baseline.py +++ b/src/tokamak_foundation_model/models/modality/video_baseline.py @@ -1,305 +1,245 @@ -import numpy as np import torch import torch.nn as nn -from base import ModalityEncoder, ModalityDecoder +import torch.nn.functional as F +from .base import ModalityEncoder, ModalityDecoder +from typing import Optional + + +# class VideoEncoder(nn.Module): +# def __init__(self, in_channels=1, n_tokens=8, token_dim=512): +# super().__init__() +# self.n_tokens = n_tokens +# self.token_dim = token_dim + +# self.net = nn.Sequential( +# nn.Conv3d(in_channels, 32, 3, padding=1), nn.ReLU(), +# nn.Conv3d(32, 64, 3, stride=(1,2,2), padding=1), nn.ReLU(), +# nn.Conv3d(64, 128, 3, stride=(1,2,2), padding=1), nn.ReLU(), +# nn.Conv3d(128, 256, 3, stride=(1,2,2), padding=1), nn.ReLU(), +# nn.Conv3d(256, token_dim, 1), nn.ReLU(), +# nn.AdaptiveAvgPool3d((n_tokens, 1, 1)), # <-- THIS must be n_tokens +# ) + +# def forward(self, x): +# # x: (B,T,H,W) -> (B,1,T,H,W) +# y = self.net(x.unsqueeze(1)) # (B,512,N,1,1) +# z = y.squeeze(-1).squeeze(-1).permute(0,2,1) # (B,N,512) +# return z + + +# class VideoDecoder(nn.Module): +# """ +# Input: z (B, N, 512) +# Output: x_hat (B, T, H, W) +# """ +# def __init__(self, out_channels: int = 1, n_tokens: int = 8, token_dim: int = 512, +# target_size=(25, 256, 256)): +# super().__init__() +# self.target_size = target_size + +# self.net = nn.Sequential( +# nn.ConvTranspose3d(token_dim, 256, kernel_size=(3, 4, 4), stride=(1, 2, 2), padding=(1, 1, 1)), +# nn.ReLU(), +# nn.ConvTranspose3d(256, 128, kernel_size=(3, 4, 4), stride=(1, 2, 2), padding=(1, 1, 1)), +# nn.ReLU(), +# nn.ConvTranspose3d(128, 64, kernel_size=(3, 4, 4), stride=(1, 2, 2), padding=(1, 1, 1)), +# nn.ReLU(), +# nn.ConvTranspose3d(64, 32, kernel_size=3, padding=1), +# nn.ReLU(), +# nn.ConvTranspose3d(32, out_channels, kernel_size=3, padding=1), +# ) +# self.refine = nn.Sequential( +# nn.Upsample(scale_factor=(1,2,2), mode="trilinear", align_corners=False), +# nn.Conv3d(1, 16, 3, padding=1), nn.ReLU(), +# nn.Upsample(scale_factor=(1,2,2), mode="trilinear", align_corners=False), +# nn.Conv3d(16, 16, 3, padding=1), nn.ReLU(), +# nn.Upsample(scale_factor=(1,2,2), mode="trilinear", align_corners=False), +# nn.Conv3d(16, 16, 3, padding=1), nn.ReLU(), +# nn.Upsample(scale_factor=(1,2,2), mode="trilinear", align_corners=False), +# nn.Conv3d(16, 16, 3, padding=1), nn.ReLU(), +# nn.Upsample(scale_factor=(1,2,2), mode="trilinear", align_corners=False), +# nn.Conv3d(16, 1, 3, padding=1), +# ) +# self.resample = nn.AdaptiveAvgPool3d(target_size) + +# def forward(self, z): +# y = z.permute(0,2,1).unsqueeze(-1).unsqueeze(-1) +# x = self.net(y) +# x = self.refine(x) # (B,1,N,256,256) +# x = torch.tanh(x) +# x = F.interpolate(x, size=self.target_size, mode="trilinear", align_corners=False) +# return x.squeeze(1) + + +# class VideoAutoEncoder(nn.Module): +# def __init__(self, n_tokens: int, target_size=(25, 256, 256), token_dim: int = 512): +# super().__init__() +# self.encoder = VideoEncoder(n_tokens=n_tokens, token_dim=token_dim) +# self.decoder = VideoDecoder(n_tokens=n_tokens, token_dim=token_dim, target_size=target_size) + +# def forward(self, x): +# z = self.encoder(x) +# x_hat = self.decoder(z) +# return x_hat, z + +# def encode(self, x): +# z = self.encoder(x) +# return z + +# def decode(self, z): +# x_hat = self.decoder(z) +# return x_hat -def create_video_test_signal( - batch_size: int = 4, - input_frames: int = 50, - frame_size: int = 256 -): +class VideoEncoder(nn.Module): """ - Create deterministic test video sequences for video encoder/decoder. - - Parameters - ---------- - batch_size : int, optional - Number of samples in batch, by default 4 - input_frames : int, optional - Number of frames per video, by default 50 - frame_size : int, optional - Height and width of each frame, by default 256 - - Returns - ------- - torch.Tensor - Test video of shape [batch_size, 1, input_frames, frame_size, frame_size] - - Notes - ----- - Test patterns per batch: - - Batch 0: Constant frame (all ones) - tests DC preservation - - Batch 1: Vertical edge (left half 0, right half 1) - tests spatial edges - - Batch 2: Single spatial impulse at center - tests spatial localization - - Batch 3: Temporal flash (single frame lit up) - tests temporal localization + Input: x (B, T, H, W) grayscale + Output: z_tokens (B, N, 512) + Also returns z_vec (B, N*512) for decoding. """ - signal = np.zeros((batch_size, 1, input_frames, frame_size, frame_size)) - - if batch_size > 0: - signal[0, 0, :, :, :] = 1.0 - if batch_size > 1: - signal[1, 0, :, :, frame_size // 2:] = 1.0 + def __init__( + self, + n_tokens: int, + token_dim: int = 512, + t_chunk: int = 25, + img_size: int = 256, + ): + super().__init__() + self.n_tokens = n_tokens + self.token_dim = token_dim + self.latent_dim = n_tokens * token_dim + + # Attached-style: stride-2 conv stack + BN + ReLU + self.enc = nn.Sequential( + nn.Conv3d(1, 16, 3, stride=2, padding=1), + nn.BatchNorm3d(16), + nn.ReLU(inplace=True), + nn.Conv3d(16, 32, 3, stride=2, padding=1), + nn.BatchNorm3d(32), + nn.ReLU(inplace=True), + nn.Conv3d(32, 64, 3, stride=2, padding=1), + nn.BatchNorm3d(64), + nn.ReLU(inplace=True), + nn.Conv3d(64, 128, 3, stride=2, padding=1), + nn.BatchNorm3d(128), + nn.ReLU(inplace=True), + nn.Conv3d(128, 256, 3, stride=2, padding=1), + nn.BatchNorm3d(256), + nn.ReLU(inplace=True), + ) - if batch_size > 2: - signal[2, 0, :, frame_size // 2, frame_size // 2] = 1.0 + # Infer flatten dim once (keeps your structure clean in notebook) + with torch.no_grad(): + dummy = torch.zeros(1, 1, t_chunk, img_size, img_size) + h = self.enc(dummy) + self._enc_shape = h.shape # (1, C0, T0, H0, W0) + flat_dim = h.flatten(1).shape[1] - if batch_size > 3: - signal[3, 0, input_frames // 2, :, :] = 1.0 + self.fc = nn.Linear(flat_dim, self.latent_dim) - return torch.from_numpy(signal).float() + def forward(self, x: torch.Tensor): + # x: (B,T,H,W) -> (B,1,T,H,W) + h = self.enc(x.unsqueeze(1)) + z_vec = self.fc(h.flatten(1)) # (B, N*512) + z_tokens = z_vec.view(x.shape[0], self.n_tokens, self.token_dim) # (B,N,512) + return z_tokens, z_vec -class VideoEncoder(nn.Module): +class VideoDecoder(nn.Module): """ - Encodes grayscale video sequences using joint 3D convolutions. - - Parameters - ---------- - input_frames : int, optional - Number of input frames (e.g., 50 for 500ms @ 100fps), by default 50 - frame_size : int, optional - Height and width of each frame (assumed square), by default 256 - d_model : int, optional - Model dimension for transformer, by default 512 - n_output_tokens : int, optional - Number of output tokens, must equal t_tokens * h_tokens * w_tokens, - by default 192 (3 * 8 * 8) - verbose : bool, optional - If True, print debug information during initialization, by default False - - Attributes - ---------- - conv_layers : nn.ModuleList - List of 3D convolutional layers - t_tokens : int - Temporal dimension of token grid (3) - h_tokens : int - Height dimension of token grid (8) - w_tokens : int - Width dimension of token grid (8) - adaptive_pool : nn.AdaptiveAvgPool3d - Adaptive pooling to exact token dimensions + Input: z_tokens (B, N, 512) OR z_vec (B, N*512) + Output: x_hat (B, T, H, W) """ def __init__( - self, - input_frames: int = 50, - frame_size: int = 256, - d_model: int = 512, - n_output_tokens: int = 192, - verbose: bool = False + self, + n_tokens: int, + token_dim: int = 512, + t_chunk: int = 25, + img_size: int = 256, + enc_shape=(1, 256, 1, 8, 8), # will be overwritten by encoder-provided shape ): super().__init__() - - self.input_frames = input_frames - self.frame_size = frame_size - self.d_model = d_model - self.n_output_tokens = n_output_tokens - self.verbose = verbose - - # Token grid: 192 = 3 × 8 × 8 - self.t_tokens = 3 - self.h_tokens = 8 - self.w_tokens = 8 - - assert self.t_tokens * self.h_tokens * self.w_tokens == n_output_tokens, ( - f"n_output_tokens ({n_output_tokens}) must equal " - f"t_tokens * h_tokens * w_tokens " - f"({self.t_tokens} * {self.h_tokens} * {self.w_tokens})" + self.n_tokens = n_tokens + self.token_dim = token_dim + self.latent_dim = n_tokens * token_dim + self.t_chunk = t_chunk + self.img_size = img_size + + # Use encoder's conv output shape to reshape back + _, C0, T0, H0, W0 = enc_shape + self.C0, self.T0, self.H0, self.W0 = C0, T0, H0, W0 + + self.fc = nn.Linear(self.latent_dim, C0 * T0 * H0 * W0) + + # Attached-style: ConvTranspose3d + BN + ReLU, final conv to 1 channel + self.dec = nn.Sequential( + nn.ConvTranspose3d(C0, 128, 3, stride=2, padding=1, output_padding=1), + nn.BatchNorm3d(128), + nn.ReLU(inplace=True), + nn.ConvTranspose3d(128, 64, 3, stride=2, padding=1, output_padding=1), + nn.BatchNorm3d(64), + nn.ReLU(inplace=True), + nn.ConvTranspose3d(64, 32, 3, stride=2, padding=1, output_padding=1), + nn.BatchNorm3d(32), + nn.ReLU(inplace=True), + nn.ConvTranspose3d(32, 16, 3, stride=2, padding=1, output_padding=1), + nn.BatchNorm3d(16), + nn.ReLU(inplace=True), + nn.ConvTranspose3d(16, 1, 3, stride=2, padding=1, output_padding=1), ) - # 3D conv stack: - # Layers 1-3: spatial stride only (preserve temporal resolution) - # Layers 4-5: joint stride (compress both space and time) - self.conv_layers = nn.ModuleList([ - # [B, 1, 50, 256, 256] → [B, 32, 50, 128, 128] - nn.Conv3d(1, 32, kernel_size=(3,7,7), stride=(1,2,2), padding=(1,3,3)), - # [B, 32, 50, 128, 128] → [B, 64, 50, 64, 64] - nn.Conv3d(32, 64, kernel_size=(3,5,5), stride=(1,2,2), padding=(1,2,2)), - # [B, 64, 50, 64, 64] → [B, 128, 50, 32, 32] - nn.Conv3d(64, 128, kernel_size=(3,5,5), stride=(1,2,2), padding=(1,2,2)), - # [B, 128, 50, 32, 32] → [B, 256, 25, 16, 16] - nn.Conv3d(128, 256, kernel_size=(3,3,3), stride=(2,2,2), padding=(1,1,1)), - # [B, 256, 25, 16, 16] → [B, d_model, 12, 8, 8] - nn.Conv3d(256, d_model, kernel_size=(3,3,3), stride=(2,2,2), padding=(1,1,1)), - ]) - - self.adaptive_pool = nn.AdaptiveAvgPool3d( - (self.t_tokens, self.h_tokens, self.w_tokens) + def forward( + self, z_tokens: torch.Tensor, z_vec: Optional[torch.Tensor] = None + ) -> torch.Tensor: + # Accept either z_tokens or z_vec + if z_vec is None: + B = z_tokens.shape[0] + z_vec = z_tokens.reshape(B, self.latent_dim) # (B, N*512) + + x = self.fc(z_vec).view( + -1, self.C0, self.T0, self.H0, self.W0 + ) # (B,C0,T0,H0,W0) + x = self.dec(x) # (B,1,T',H',W') + + # Force exact output size (like the attached code typically does) + x = F.interpolate( + x, + size=(self.t_chunk, self.img_size, self.img_size), + mode="trilinear", + align_corners=False, ) - self.activation = nn.GELU() - self.norm = nn.LayerNorm(d_model) - - if self.verbose: - print(f"VideoEncoder:") - print(f" Input: [B, 1, {input_frames}, {frame_size}, {frame_size}]") - print(f" Conv1: [B, 32, 50, 128, 128]") - print(f" Conv2: [B, 64, 50, 64, 64]") - print(f" Conv3: [B, 128, 50, 32, 32]") - print(f" Conv4: [B, 256, 25, 16, 16]") - print(f" Conv5: [B, {d_model}, 12, 8, 8]") - print(f" Pool: [B, {d_model}, {self.t_tokens}, {self.h_tokens}, {self.w_tokens}]") - print(f" Output: [B, {n_output_tokens}, {d_model}]") - - def forward(self, x): - """ - Encode video sequence into tokens. - - Parameters - ---------- - x : torch.Tensor - Input video of shape [batch, 1, input_frames, frame_size, frame_size] - - Returns - ------- - torch.Tensor - Encoded tokens of shape [batch, n_output_tokens, d_model] - """ - B = x.shape[0] - - for conv in self.conv_layers: - x = self.activation(conv(x)) - - x = self.adaptive_pool(x) # [B, d_model, t_tokens, h_tokens, w_tokens] - x = x.flatten(2) # [B, d_model, n_output_tokens] - x = x.transpose(1, 2) # [B, n_output_tokens, d_model] - x = self.norm(x) - - return x + # If your input is normalized to [0,1], keep sigmoid: + x = torch.sigmoid(x) + + return x.squeeze(1) # (B,T,H,W) -class VideoDecoder(nn.Module): - """ - Mirrors VideoEncoder for pre-training via masked autoencoding. - Reconstructs the original input video from encoder tokens. - - Parameters - ---------- - input_frames : int, optional - Number of frames to reconstruct (must match VideoEncoder.input_frames), - by default 50 - frame_size : int, optional - Height and width of frames to reconstruct, by default 256 - d_model : int, optional - Model dimension from encoder, by default 512 - n_input_tokens : int, optional - Number of input tokens from encoder, by default 192 - verbose : bool, optional - If True, print debug information during initialization, by default False - - Attributes - ---------- - deconv_layers : nn.ModuleList - List of 3D transposed convolutional layers mirroring encoder - adaptive_pool : nn.AdaptiveAvgPool3d - Ensures exact output dimensions - """ +class VideoAutoEncoder(nn.Module): def __init__( self, - input_frames: int = 50, - frame_size: int = 256, - d_model: int = 512, - n_input_tokens: int = 192, - verbose: bool = False + n_tokens: int, + t_chunk: int = 25, + img_size: int = 256, + token_dim: int = 512, ): super().__init__() - - self.input_frames = input_frames - self.frame_size = frame_size - self.d_model = d_model - self.n_input_tokens = n_input_tokens - self.verbose = verbose - - # Starting spatiotemporal dimensions (mirrors encoder adaptive pool output) - self.t_start = 3 - self.h_start = 8 - self.w_start = 8 - - assert self.t_start * self.h_start * self.w_start == n_input_tokens, ( - f"n_input_tokens ({n_input_tokens}) must equal " - f"t_start * h_start * w_start " - f"({self.t_start} * {self.h_start} * {self.w_start})" + self.encoder = VideoEncoder( + n_tokens=n_tokens, token_dim=token_dim, t_chunk=t_chunk, img_size=img_size ) - # Mirror encoder in reverse: - # Layers 1-2: joint upsample - # Layers 3-5: spatial upsample only - self.deconv_layers = nn.ModuleList([ - # [B, d_model, 3, 8, 8] → [B, 256, 6, 16, 16] - nn.ConvTranspose3d(d_model, 256, kernel_size=(3,3,3), stride=(2,2,2), - padding=(1,1,1), output_padding=(1,1,1)), - # [B, 256, 6, 16, 16] → [B, 128, 12, 32, 32] - nn.ConvTranspose3d(256, 128, kernel_size=(3,3,3), stride=(2,2,2), - padding=(1,1,1), output_padding=(1,1,1)), - # [B, 128, 12, 32, 32] → [B, 64, 12, 64, 64] - nn.ConvTranspose3d(128, 64, kernel_size=(3,5,5), stride=(1,2,2), - padding=(1,2,2), output_padding=(0,1,1)), - # [B, 64, 12, 64, 64] → [B, 32, 12, 128, 128] - nn.ConvTranspose3d(64, 32, kernel_size=(3,5,5), stride=(1,2,2), - padding=(1,2,2), output_padding=(0,1,1)), - # [B, 32, 12, 128, 128] → [B, 1, 12, 256, 256] - nn.ConvTranspose3d(32, 1, kernel_size=(3,7,7), stride=(1,2,2), - padding=(1,3,3), output_padding=(0,1,1)), - ]) - - self.adaptive_pool = nn.AdaptiveAvgPool3d( - (input_frames, frame_size, frame_size) + # Build decoder using encoder's inferred shape + self.decoder = VideoDecoder( + n_tokens=n_tokens, + token_dim=token_dim, + t_chunk=t_chunk, + img_size=img_size, + enc_shape=self.encoder._enc_shape, ) - self.activation = nn.GELU() - - if self.verbose: - print(f"VideoDecoder:") - print(f" Input: [B, {n_input_tokens}, {d_model}]") - print(f" Reshape: [B, {d_model}, {self.t_start}, {self.h_start}, {self.w_start}]") - print(f" Deconv1: [B, 256, 6, 16, 16]") - print(f" Deconv2: [B, 128, 12, 32, 32]") - print(f" Deconv3: [B, 64, 12, 64, 64]") - print(f" Deconv4: [B, 32, 12, 128, 128]") - print(f" Deconv5: [B, 1, 12, 256, 256]") - print(f" Pool: [B, 1, {input_frames}, {frame_size}, {frame_size}]") - - def forward(self, x): - """ - Encode video sequence into tokens. - - Parameters - ---------- - x : torch.Tensor - Input video of shape [batch, 1, input_frames, frame_size, frame_size] - - Returns - ------- - torch.Tensor - Encoded tokens of shape [batch, n_output_tokens, d_model] - """ - B = x.shape[0] - - for conv in self.conv_layers: - x = self.activation(conv(x)) - - x = self.adaptive_pool(x) # [B, d_model, t_tokens, h_tokens, w_tokens] - x = x.flatten(2) # [B, d_model, n_output_tokens] - x = x.transpose(1, 2) # [B, n_output_tokens, d_model] - x = self.norm(x) - - return x - - -if __name__ == "__main__": - - print("=" * 60) - print("VideoEncoder / VideoDecoder") - print("=" * 60) - vid_enc = VideoEncoder(input_frames=50, frame_size=256, - d_model=512, n_output_tokens=192, verbose=True) - vid_dec = VideoDecoder(input_frames=50, frame_size=256, - d_model=512, n_input_tokens=192, verbose=True) - x_vid = create_video_test_signal() - tokens_vid = vid_enc(x_vid) - recon_vid = vid_dec(tokens_vid) - print(f"Input: {x_vid.shape}") # [4, 1, 50, 256, 256] - print(f"Tokens: {tokens_vid.shape}") # [4, 192, 512] - print(f"Recon: {recon_vid.shape}") # [4, 1, 50, 256, 256] + + def forward(self, x: torch.Tensor): + z_tokens, z_vec = self.encoder(x) + x_hat = self.decoder(z_tokens, z_vec=z_vec) + return x_hat, z_tokens \ No newline at end of file From 16e9bfbcbb7e5856f2a3150cf384507ca0f23b9a Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Fri, 13 Feb 2026 20:11:57 -0500 Subject: [PATCH 05/30] Minor changes in the example scripts. More preprocessing options for the dataset class. --- scripts/actuator_reconstruction.py | 66 +++++++++++++++++++ scripts/fast_time_series_reconstruction.py | 32 +++------ .../data/data_loader.py | 16 +++-- .../models/modality/profile_baseline.py | 4 +- .../models/modality/time_series_baseline.py | 18 +++-- 5 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 scripts/actuator_reconstruction.py diff --git a/scripts/actuator_reconstruction.py b/scripts/actuator_reconstruction.py new file mode 100644 index 0000000..eabecd3 --- /dev/null +++ b/scripts/actuator_reconstruction.py @@ -0,0 +1,66 @@ +from pathlib import Path +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.models.modality.fast_time_series_baseline import ( + TimeSeriesAutoencoder) +from tokamak_foundation_model.trainer.trainer import UnimodalTrainer + + +def worker_init_fn(worker_id): + """Each worker needs to open its own file handle.""" + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None: + dataset = worker_info.dataset + # Force re-open file for this worker + if hasattr(dataset, 'datasets'): # ConcatDataset + for ds in dataset.datasets: + ds.h5_file = None + ds._open_hdf5() + else: + dataset.h5_file = None + dataset._open_hdf5() + + +hdf5_files = sorted( + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") +) +stats = torch.load( + Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") +) + +datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + chunk_duration_s=0.7, + input_signals=["tin", ], + target_signals=["tin", ], + prediction_mode=False, + ) + for f in hdf5_files +] + +concatenated_dataset = ConcatDataset(datasets_processed) + +dataloader = DataLoader( + concatenated_dataset, + batch_size=8, + shuffle=False, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn + ) + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +model = TimeSeriesAutoencoder(n_channels=8, input_length=7000, n_tokens=140) +model = model.to(device) +loss_fn = nn.MSELoss() +optimizer = optim.AdamW(model.parameters(), lr=0.005) +trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50, + checkpoint_path='checkpoint_tin.pth') +# ECH and gas are critical +trainer.train(dataloader, val_dataloader=dataloader, modality_key="tin") diff --git a/scripts/fast_time_series_reconstruction.py b/scripts/fast_time_series_reconstruction.py index e0dd2d4..6fd16fd 100644 --- a/scripts/fast_time_series_reconstruction.py +++ b/scripts/fast_time_series_reconstruction.py @@ -6,25 +6,10 @@ from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn from tokamak_foundation_model.models.modality.fast_time_series_baseline import ( - TimeSeriesEncoder, TimeSeriesDecoder) + TimeSeriesAutoencoder) from tokamak_foundation_model.trainer.trainer import UnimodalTrainer -class DummyModel(torch.nn.Module): - def __init__(self): - super(DummyModel, self).__init__() - self.encoder = TimeSeriesEncoder( - kernel_size=11, n_channels=8, input_length=5000, d_model=512, - n_output_tokens=100) - self.decoder = TimeSeriesDecoder( - kernel_size=11, n_channels=8, input_length=5000, d_model=512, - n_input_tokens=100) - - def forward(self, x): - x_encoded = self.encoder(x) - return self.decoder(x_encoded) - - def worker_init_fn(worker_id): """Each worker needs to open its own file handle.""" worker_info = torch.utils.data.get_worker_info() @@ -40,9 +25,6 @@ def worker_init_fn(worker_id): dataset._open_hdf5() -model = DummyModel() - - hdf5_files = sorted( Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") ) @@ -54,8 +36,8 @@ def worker_init_fn(worker_id): TokamakH5Dataset( hdf5_path=str(f), preprocessing_stats=stats, - input_signals=["pin", ], - target_signals=["pin", ], + input_signals=["d_alpha", ], + target_signals=["d_alpha", ], prediction_mode=False, ) for f in hdf5_files @@ -71,9 +53,11 @@ def worker_init_fn(worker_id): worker_init_fn=worker_init_fn ) -optimizer = optim.AdamW(model.parameters(), lr=0.005) -loss_fn = nn.MSELoss() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +model = TimeSeriesAutoencoder() model = model.to(device) +loss_fn = nn.MSELoss() +optimizer = optim.AdamW(model.parameters(), lr=0.005) trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50) -trainer.train(dataloader, val_dataloader=dataloader, modality_key="pin") +trainer.train(dataloader, val_dataloader=dataloader, modality_key="d_alpha") diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index ebb4583..2f7023a 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -158,7 +158,7 @@ class TokamakH5Dataset(Dataset): 6, 10e3, apply_stft=False, - preprocess=PreprocessConfig(method="standardize"), + preprocess=PreprocessConfig(method="none"), ), SignalConfig( "gas", @@ -166,7 +166,7 @@ class TokamakH5Dataset(Dataset): 5, 10e3, apply_stft=False, - preprocess=PreprocessConfig(method="standardize"), + preprocess=PreprocessConfig(method="none"), ), SignalConfig( "ech", @@ -174,7 +174,7 @@ class TokamakH5Dataset(Dataset): 11, 10e3, apply_stft=False, - preprocess=PreprocessConfig(method="standardize"), + preprocess=PreprocessConfig(method="none"), ), SignalConfig( "pin", @@ -190,7 +190,7 @@ class TokamakH5Dataset(Dataset): 8, 10e3, apply_stft=False, - preprocess=PreprocessConfig(method="standardize"), + preprocess=PreprocessConfig(method="none"), ), SignalConfig( "mse", @@ -206,7 +206,7 @@ class TokamakH5Dataset(Dataset): 44, 1e2, apply_stft=False, - preprocess=PreprocessConfig(method="none"), + preprocess=PreprocessConfig(method="log"), ), ] @@ -332,7 +332,7 @@ def _apply_preprocessing( return (tensor - min_val) / (max_val - min_val + config.eps) elif config.method == "log_standardize": - tensor_log = torch.log(tensor + 1) + tensor_log = torch.log10(tensor + 1) if config.mean is None or config.std is None: print("Warning: log_standardize requested but no statistics provided") @@ -350,6 +350,10 @@ def _apply_preprocessing( return (tensor_log - mean) / (std + config.eps) + elif config.method == "log": + tensor_log = torch.log10(tensor + 1) + return tensor_log + return tensor def _compute_duration_from_handle(self, f: h5py.File) -> float: diff --git a/src/tokamak_foundation_model/models/modality/profile_baseline.py b/src/tokamak_foundation_model/models/modality/profile_baseline.py index 1e7b1eb..ded395d 100644 --- a/src/tokamak_foundation_model/models/modality/profile_baseline.py +++ b/src/tokamak_foundation_model/models/modality/profile_baseline.py @@ -101,7 +101,7 @@ def __init__( n_time_points: int = 50, d_model: int = 512, n_output_tokens: int = 10, - kernel_size: int = 5, + kernel_size: int = 3, verbose: bool = False, ): super().__init__() @@ -119,8 +119,10 @@ def __init__( # Spatial MLP: encodes each time step's spatial profile self.spatial_encoder = nn.Sequential( nn.Linear(n_spatial_points, 128), + nn.InstanceNorm1d(128), self.activation, nn.Linear(128, 256), + nn.InstanceNorm1d(256), self.activation, nn.Linear(256, d_model) ) diff --git a/src/tokamak_foundation_model/models/modality/time_series_baseline.py b/src/tokamak_foundation_model/models/modality/time_series_baseline.py index 63963b3..f7e7055 100644 --- a/src/tokamak_foundation_model/models/modality/time_series_baseline.py +++ b/src/tokamak_foundation_model/models/modality/time_series_baseline.py @@ -7,9 +7,15 @@ class TimeSeriesEncoder(ModalityEncoder): def __init__(self, in_channels, out_features=64): super().__init__(in_channels, out_features) self.net = nn.Sequential( - nn.Conv1d(in_channels, 32, 3, padding=1), nn.ReLU(), nn.MaxPool1d(2), - nn.Conv1d(32, 64, 3, padding=1), nn.ReLU(), nn.AdaptiveAvgPool1d(1), - nn.Flatten(), nn.Linear(64, out_features), nn.ReLU(), + nn.Conv1d(in_channels, 32, 3, padding=1), + nn.ReLU(), + nn.MaxPool1d(2), + nn.Conv1d(32, 64, 3, padding=1), + nn.ReLU(), + nn.AdaptiveAvgPool1d(1), + nn.Flatten(), + nn.Linear(64, out_features), + nn.ReLU(), ) def forward(self, x): @@ -21,9 +27,11 @@ def __init__(self, in_features=64, out_channels=1, target_length=100): super().__init__(in_features, out_channels) self.target_length = target_length self.net = nn.Sequential( - nn.Linear(in_features, 64), nn.ReLU(), + nn.Linear(in_features, 64), + nn.ReLU(), nn.Unflatten(1, (64, 1)), - nn.ConvTranspose1d(64, 32, 4, stride=2, padding=1), nn.ReLU(), + nn.ConvTranspose1d(64, 32, 4, stride=2, padding=1), + nn.ReLU(), nn.ConvTranspose1d(32, out_channels, 4, stride=2, padding=1), ) self.resample = nn.AdaptiveAvgPool1d(target_length) From 475a6e2b4cc54e7a975b76946466f5cfa993e1a4 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Sat, 14 Feb 2026 16:21:32 -0500 Subject: [PATCH 06/30] Fixed a bug where the dataset class failed when using multiple workers and opening an H5 file prior to distributing the dataset across all workers. Significant updates in the Fast time series baseline and actuator reconstruction classes. --- scripts/actuator_reconstruction.py | 222 ++++++++++--- scripts/standardize_dataset.py | 2 +- scripts/train_unimodal_autoencoder.py | 164 ++++------ .../data/data_loader.py | 37 ++- src/tokamak_foundation_model/data/utils.py | 16 + .../models/modality/actuator_baseline.py | 158 ++++----- .../modality/fast_time_series_baseline.py | 303 +++++++++++++----- .../models/model_factory.py | 44 +++ 8 files changed, 612 insertions(+), 334 deletions(-) create mode 100644 src/tokamak_foundation_model/data/utils.py create mode 100644 src/tokamak_foundation_model/models/model_factory.py diff --git a/scripts/actuator_reconstruction.py b/scripts/actuator_reconstruction.py index eabecd3..0af3da8 100644 --- a/scripts/actuator_reconstruction.py +++ b/scripts/actuator_reconstruction.py @@ -1,66 +1,182 @@ from pathlib import Path +import argparse +import logging + import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import ConcatDataset, DataLoader from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn -from tokamak_foundation_model.models.modality.fast_time_series_baseline import ( - TimeSeriesAutoencoder) +from tokamak_foundation_model.data.utils import worker_init_fn from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) +from tokamak_foundation_model.utils import DefaultDrawer -def worker_init_fn(worker_id): - """Each worker needs to open its own file handle.""" - worker_info = torch.utils.data.get_worker_info() - if worker_info is not None: - dataset = worker_info.dataset - # Force re-open file for this worker - if hasattr(dataset, 'datasets'): # ConcatDataset - for ds in dataset.datasets: - ds.h5_file = None - ds._open_hdf5() - else: - dataset.h5_file = None - dataset._open_hdf5() - - -hdf5_files = sorted( - Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") -) -stats = torch.load( - Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") -) - -datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - chunk_duration_s=0.7, - input_signals=["tin", ], - target_signals=["tin", ], - prediction_mode=False, - ) - for f in hdf5_files -] - -concatenated_dataset = ConcatDataset(datasets_processed) - -dataloader = DataLoader( - concatenated_dataset, - batch_size=8, - shuffle=False, - collate_fn=collate_fn, - worker_init_fn=worker_init_fn - ) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = TimeSeriesAutoencoder(n_channels=8, input_length=7000, n_tokens=140) -model = model.to(device) -loss_fn = nn.MSELoss() -optimizer = optim.AdamW(model.parameters(), lr=0.005) -trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50, - checkpoint_path='checkpoint_tin.pth') -# ECH and gas are critical -trainer.train(dataloader, val_dataloader=dataloader, modality_key="tin") +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser.add_argument( + "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + default="pin", + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--hop_length", type=int, default=256, help="Hop length for STFT.", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default="actuator", + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=512, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=140, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=1, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=50, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=1e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.05, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + signal_name = args.signal + model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Signal: {signal_name}, Model: {model_name}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=False, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + # Not sure if this is elegant + sample_data = next(iter(concatenated_dataset))[signal_name] + n_channels = sample_data.shape[0] + logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") + + ### Model Setup ### + model = build_model(model_name, n_channels, args.d_model, args.n_tokens).to(device) + + n_params = sum(p.numel() for p in model.parameters()) + logger.info(f"Model parameters: {n_params:,}") + + optimizer = optim.AdamW( + model.parameters(), + lr=args.lr, + ) + # loss_fn = nn.L1Loss() + loss_fn = nn.MSELoss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = UnimodalTrainer( + epochs=args.epochs, + checkpoint_path=checkpoint_path, + model=model, + optimizer=optimizer, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=checkpoint_path) + + trainer.train(dataloader, modality_key=signal_name) + + +if __name__ == "__main__": + main() diff --git a/scripts/standardize_dataset.py b/scripts/standardize_dataset.py index 61a246b..cc8f1fe 100644 --- a/scripts/standardize_dataset.py +++ b/scripts/standardize_dataset.py @@ -4,7 +4,7 @@ hdf5_files = sorted( Path( - "C:/Users/admin/PycharmProjects/nstx/foundation_model_notes/tokamak_package/" + "C:/Users/admin/PycharmProjects/FusionAIHub/scripts/" ).glob("*_processed.h5") ) all_input_signals = [ diff --git a/scripts/train_unimodal_autoencoder.py b/scripts/train_unimodal_autoencoder.py index 3001337..efd9175 100644 --- a/scripts/train_unimodal_autoencoder.py +++ b/scripts/train_unimodal_autoencoder.py @@ -8,17 +8,12 @@ from torch.utils.data import ConcatDataset, DataLoader from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.data.utils import worker_init_fn from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) from tokamak_foundation_model.utils import DefaultDrawer -from tokamak_foundation_model.models.modality import ( - ActuatorBaselineAutoEncoder, - SlowTimeSeriesBaselineAutoEncoder, - FastTimeSeriesBaselineAutoEncoder, - SpatialProfileBaselineAutoEncoder, - SpectrogramBaselineAutoEncoder, - VideoBaselineAutoEncoder, -) # TODO: Add ddp support device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -26,95 +21,76 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -### Signal-to-model default mapping ### - -SIGNAL_MODEL_DEFAULTS = { - "gas": "actuator", - "ech": "actuator", - "pin": "actuator", - "tin": "actuator", - "d_alpha": "fast_time_series", - "mse": "profile", - "ts_core_density": "profile", - "mhr": "spectrogram", - "ece": "spectrogram", - "co2": "spectrogram", - "bolo": "video", - "irtv": "video", - "tangtv": "video", -} - -MODEL_REGISTRY = { - "actuator": ActuatorBaselineAutoEncoder, - "fast_time_series": FastTimeSeriesBaselineAutoEncoder, - "slow_time_series": SlowTimeSeriesBaselineAutoEncoder, - "profile": SpatialProfileBaselineAutoEncoder, - "spectrogram": SpectrogramBaselineAutoEncoder, - "video": VideoBaselineAutoEncoder, -} - - -# TODO: Move into source code -def build_model(model_name, n_channels, d_model, n_tokens): - """Build the appropriate autoencoder. - - All autoencoders share the same interface: (n_channels, d_model, n_tokens). - """ - cls = MODEL_REGISTRY[model_name] - kwargs = dict(n_channels=n_channels, d_model=d_model) - if n_tokens is not None: kwargs["n_tokens"] = n_tokens - return cls(**kwargs) - -# TODO: Move to data loader -def worker_init_fn(worker_id): - worker_info = torch.utils.data.get_worker_info() - if worker_info is not None: - dataset = worker_info.dataset - if hasattr(dataset, 'datasets'): - for ds in dataset.datasets: - ds.h5_file = None - ds._open_hdf5() - else: - dataset.h5_file = None - dataset._open_hdf5() - def main(): ### Settings ### parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") - parser.add_argument("--signal", required=True, choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - help="Signal name to train on") - parser.add_argument("--model", choices=list(MODEL_REGISTRY.keys()), default=None, - help="Model type (default: auto-selected from signal)") - parser.add_argument("--data_dir", type=str, - default="/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data", - help="Path to HDF5 data directory") - parser.add_argument("--stats_path", type=str, default="data/preprocessing_stats.pt", - help="Path to preprocessing stats file") - parser.add_argument("--d_model", type=int, default=64, help="Model dimension") - parser.add_argument("--n_tokens", type=int, default=None, - help="Number of latent tokens (default: use model default)") - parser.add_argument("--batch_size", type=int, default=2, - help="Batch size (for spectrograms, each sample's C channels are " - "processed independently, so effective batch = batch_size * C)") - parser.add_argument("--num_workers", type=int, default=4, help="Number of data loader workers") - parser.add_argument("--epochs", type=int, default=10, help="Number of training epochs") - parser.add_argument("--lr", type=float, default=1e-3, help="Learning rate") - parser.add_argument("--weight_decay", type=float, default=0.05, - help="AdamW weight decay") - parser.add_argument("--warmup_epochs", type=int, default=5, - help="LR warmup epochs (0 to disable scheduler)") - parser.add_argument("--min_lr", type=float, default=0.0, - help="Minimum LR at end of cosine decay") - parser.add_argument("--checkpoint_dir", type=str, default="runs", - help="Directory for checkpoints") - parser.add_argument("--num_plots", type=int, default=4, - help="Number of reconstruction plots per epoch") - parser.add_argument("--log_interval", type=int, default=1, - help="Plot every N epochs") - parser.add_argument("--resume", action="store_true", default=False, - help="Resume training from checkpoint") + parser.add_argument( + "--signal", required=True, choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default=None, + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, default="data/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=64, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=None, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=4, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=10, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=1e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.05, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) args = parser.parse_args() ### Paths ### @@ -122,7 +98,9 @@ def main(): model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] data_dir = Path(args.data_dir) statistics_path = Path(args.stats_path) - checkpoint_path = Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) checkpoint_path.parent.mkdir(parents=True, exist_ok=True) logger.info(f"Signal: {signal_name}, Model: {model_name}") diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index e8bb3d1..cfa697e 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import Optional import torch.nn.functional as F +import copy def compute_preprocessing_stats( @@ -228,6 +229,10 @@ def __init__( input_signals: Optional[list[str]] = None, target_signals: Optional[list[str]] = None, ): + # Make instance-level copies to avoid class-level mutation + self.signal_configs = copy.deepcopy(self.SIGNAL_CONFIGS) + self.movie_configs = copy.deepcopy(self.MOVIE_CONFIGS) + self.hdf5_path = Path(hdf5_path) self.chunk_duration_s = chunk_duration_s self.n_fft = n_fft @@ -262,7 +267,7 @@ def __init__( def _update_preprocessing_stats(self): """Update preprocessing configs with loaded statistics.""" - for config in self.SIGNAL_CONFIGS: + for config in self.signal_configs: if config.name in self.preprocessing_stats: stats = self.preprocessing_stats[config.name] if "mean" in stats: @@ -332,7 +337,7 @@ def _apply_preprocessing( return (tensor - min_val) / (max_val - min_val + config.eps) elif config.method == "log_standardize": - tensor_log = torch.log10(tensor + 1) + tensor_log = torch.log(tensor + 1) if config.mean is None or config.std is None: print("Warning: log_standardize requested but no statistics provided") @@ -350,10 +355,6 @@ def _apply_preprocessing( return (tensor_log - mean) / (std + config.eps) - elif config.method == "log": - tensor_log = torch.log10(tensor + 1) - return tensor_log - return tensor def _compute_duration_from_handle(self, f: h5py.File) -> float: @@ -482,7 +483,7 @@ def _compute_stft(self, signal: torch.Tensor) -> torch.Tensor: window=self.stft_window, return_complex=True, ) - spec = spec[:, 1:, :] # Remove DC component (extreme values) + spec = spec[:, 1:, :] # Remove DC component (extreme values) return torch.abs(spec) def _load_metadata(self, f: h5py.File) -> dict: @@ -504,6 +505,16 @@ def _load_metadata(self, f: h5py.File) -> dict: def __len__(self): return self.length + def __getstate__(self): + """Prepare state for pickling - exclude HDF5 file handle.""" + state = self.__dict__.copy() + state['h5_file'] = None + return state + + def __setstate__(self, state): + """Restore state after unpickling.""" + self.__dict__.update(state) + def _process_signal( self, data: torch.Tensor, config: SignalConfig ) -> torch.Tensor: @@ -636,14 +647,14 @@ def _getitem_standard(self, idx): # Load and process all signals all_signals = {} - for config in self.SIGNAL_CONFIGS: + for config in self.signal_configs: if config.name in self.input_signals: raw_data = self._load_signal_raw(self.h5_file, config, t_start, t_end) all_signals[config.name] = self._process_signal(raw_data, config) # Load and process movies all_movies = {} - for movie_config in self.MOVIE_CONFIGS: + for movie_config in self.movie_configs: if movie_config.name in self.input_signals: raw_movie = self._load_movie_raw( self.h5_file, movie_config, t_start, t_end @@ -668,7 +679,7 @@ def _getitem_prediction(self, idx): # Load and process all signals with extended window all_signals = {} - for config in self.SIGNAL_CONFIGS: + for config in self.signal_configs: if config.name not in signals_to_load: continue raw_data = self._load_signal_raw(self.h5_file, config, t_start, t_end) @@ -676,7 +687,7 @@ def _getitem_prediction(self, idx): # Load and process movies all_movies = {} - for movie_config in self.MOVIE_CONFIGS: + for movie_config in self.movie_configs: if movie_config.name not in signals_to_load: continue # Load raw movie data @@ -691,7 +702,7 @@ def _getitem_prediction(self, idx): targets = {} # For signals: split at input_frames - for config in self.SIGNAL_CONFIGS: + for config in self.signal_configs: signal = all_signals[config.name] if config.apply_stft: @@ -708,7 +719,7 @@ def _getitem_prediction(self, idx): targets[config.name] = signal[..., n_training_frames:] # Movies: split along time dimension - for movie_config in self.MOVIE_CONFIGS: + for movie_config in self.movie_configs: movie_name = movie_config.name movie_data = all_movies[movie_name] n_training_frames = round(self.chunk_duration_s * movie_config.target_fps) diff --git a/src/tokamak_foundation_model/data/utils.py b/src/tokamak_foundation_model/data/utils.py new file mode 100644 index 0000000..6cf7acd --- /dev/null +++ b/src/tokamak_foundation_model/data/utils.py @@ -0,0 +1,16 @@ +import torch + + +def worker_init_fn(worker_id): + """Each worker needs to open its own file handle.""" + worker_info = torch.utils.data.get_worker_info() + if worker_info is not None: + dataset = worker_info.dataset + # Force re-open file for this worker + if hasattr(dataset, 'datasets'): # ConcatDataset + for ds in dataset.datasets: + ds.h5_file = None + ds._open_hdf5() + else: + dataset.h5_file = None + dataset._open_hdf5() diff --git a/src/tokamak_foundation_model/models/modality/actuator_baseline.py b/src/tokamak_foundation_model/models/modality/actuator_baseline.py index 3c3bd25..06e62f8 100644 --- a/src/tokamak_foundation_model/models/modality/actuator_baseline.py +++ b/src/tokamak_foundation_model/models/modality/actuator_baseline.py @@ -2,112 +2,72 @@ import torch.nn as nn import torch.nn.functional as F -from .base import ModalityEncoder, ModalityDecoder, ModalityAutoEncoder +from .fast_time_series_baseline import (FastTimeSeriesBaselineEncoder, + FastTimeSeriesBaselineDecoder, + FastTimeSeriesBaselineAutoEncoder) -class ActuatorBaselineEncoder(ModalityEncoder): +class ActuatorBaselineEncoder(FastTimeSeriesBaselineEncoder): - def __init__(self, - n_channels: int, - d_model: int = 64, - n_tokens: int = 0, + def __init__( + self, + n_channels: int, + d_model: int = 512, + n_tokens: int = 100, + input_length: int = 5000, + n_conv_layers: int = 4, + kernel_size: int = 3, ): - super().__init__(n_channels, d_model, n_tokens) - - self.n_conv_layers = 3 - self.kernel_size = 7 - - intermediate = [min(32 * (2 ** i), d_model) for i in range(self.n_conv_layers - 1)] - channels = [n_channels] + intermediate + [d_model] - - self.conv_layers = nn.ModuleList([ - nn.Conv1d( - in_channels=channels[i], - out_channels=channels[i + 1], - kernel_size=self.kernel_size, - padding=self.kernel_size // 2, - ) - for i in range(self.n_conv_layers) - ]) - - if n_tokens > 0: - self.adaptive_pool = nn.AdaptiveAvgPool1d(n_tokens) - - self.activation = nn.GELU() - self.norm = nn.LayerNorm(d_model) - - def forward(self, x): - B, C, T = x.shape - - for conv in self.conv_layers: - x = self.activation(conv(x)) - - if self.n_tokens > 0: - x = self.adaptive_pool(x) # [B, d_model, n_tokens] - - x = x.transpose(1, 2) # [B, n_tokens, d_model] - x = self.norm(x) - - return x - - -class ActuatorBaselineDecoder(ModalityDecoder): - - def __init__(self, - n_channels: int, - d_model: int = 64, + super().__init__( + n_channels, + d_model, + n_tokens, + input_length, + n_conv_layers, + kernel_size + ) + + +class ActuatorBaselineDecoder(FastTimeSeriesBaselineDecoder): + + def __init__( + self, + n_channels: int = 6, + input_length: int = 5000, + d_model: int = 512, + n_tokens: int = 100, + n_deconv_layers: int = 4, + kernel_size: int = 3, ): - super().__init__(n_channels, d_model) - - self.n_deconv_layers = 3 - self.kernel_size = 7 - - # Mirror encoder channel progression (reversed) - intermediate = [min(32 * (2 ** i), d_model) for i in range(self.n_deconv_layers - 1)] - channels = [d_model] + list(reversed(intermediate)) + [n_channels] - - self.deconv_layers = nn.ModuleList([ - nn.ConvTranspose1d( - in_channels=channels[i], - out_channels=channels[i + 1], - kernel_size=self.kernel_size, - padding=self.kernel_size // 2, - ) - for i in range(self.n_deconv_layers) - ]) - - self.activation = nn.GELU() - - def forward(self, z, output_shape=None): - B, D, T = z.shape - - z = z.transpose(1, 2) # [B, d_model, n_tokens] - - for i, deconv in enumerate(self.deconv_layers): - z = deconv(z) - if i < len(self.deconv_layers) - 1: - z = self.activation(z) - - if output_shape is not None: - z = F.adaptive_avg_pool1d(z, output_shape[-1]) - - return z - - -class ActuatorBaselineAutoEncoder(ModalityAutoEncoder): - - def __init__(self, - n_channels: int, - d_model: int = 64, - n_tokens: int = 0, + super().__init__( + n_channels, + input_length, + d_model, + n_tokens, + n_deconv_layers, + kernel_size + ) + + +class ActuatorBaselineAutoEncoder(FastTimeSeriesBaselineAutoEncoder): + def __init__( + self, + n_channels: int = 6, + input_length: int = 5000, + d_model: int = 512, + n_tokens: int = 100, + n_layers: int = 4, + kernel_size: int = 3, ): - super().__init__(n_channels, d_model, n_tokens) - self.encoder = ActuatorBaselineEncoder(n_channels, d_model, n_tokens) - self.decoder = ActuatorBaselineDecoder(n_channels, d_model) + super().__init__( + n_channels, + input_length, + d_model, + n_tokens, + n_layers, + kernel_size + ) - def forward(self, x): - output_shape = x.shape[:-1] - return self.decoder(self.encoder(x), output_shape=output_shape) if __name__ == "__main__": diff --git a/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py b/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py index 14209f9..e92df59 100644 --- a/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py +++ b/src/tokamak_foundation_model/models/modality/fast_time_series_baseline.py @@ -7,96 +7,249 @@ class FastTimeSeriesBaselineEncoder(ModalityEncoder): + """ + Encodes fast time-series diagnostics using strided 1D convolutions. - def __init__(self, in_channels, out_features=64, hidden_dim=128): - super().__init__(in_channels, out_features) - self.conv_layers = nn.Sequential( - # Layer 1: (B, C, T) -> (B, 64, T//5) - nn.Conv1d(in_channels, 64, kernel_size=10, stride=5, padding=2), - nn.GroupNorm(8, 64), - nn.GELU(), - # Layer 2: -> (B, 128, T//15) - nn.Conv1d(64, hidden_dim, kernel_size=5, stride=3, padding=1), - nn.GroupNorm(16, hidden_dim), - nn.GELU(), - # Layer 3: -> (B, 256, T//30) - nn.Conv1d(hidden_dim, hidden_dim * 2, kernel_size=3, stride=2, padding=1), - nn.GroupNorm(16, hidden_dim * 2), - nn.GELU(), - # Layer 4: -> (B, 256, T//60) - nn.Conv1d(hidden_dim * 2, hidden_dim * 2, kernel_size=3, stride=2, padding=1), - nn.GroupNorm(16, hidden_dim * 2), - nn.GELU(), - ) - self.pool = nn.AdaptiveAvgPool1d(1) - self.proj = nn.Sequential( - nn.Flatten(), - nn.Linear(hidden_dim * 2, out_features), - nn.ReLU(), - ) + Parameters + ---------- + n_channels : int, optional + Number of input channels (e.g., 6 for filterscopes), by default 6 + input_length : int, optional + Length of input time series (e.g., 5000 for 500ms @ 10kHz), by default 5000 + d_model : int, optional + Model dimension for transformer, by default 512 + n_output_tokens : int, optional + Number of temporal tokens to output, by default 100 + n_conv_layers : int, optional + Number of convolutional layers, by default 4 + kernel_size : int, optional + Kernel size for convolutions, by default 15 + + Attributes + ---------- + stride : int + Calculated stride for convolutions based on desired compression ratio + channels : list of int + Channel sizes at each layer, dynamically computed + conv_layers : nn.ModuleList + List of 1D convolutional layers + adaptive_pool : nn.AdaptiveAvgPool1d + Adaptive pooling layer to ensure exact output token count + """ + + def __init__( + self, + n_channels: int, + d_model: int = 512, + n_tokens: int = 100, + input_length: int = 5000, + n_conv_layers: int = 4, + kernel_size: int = 3, + ): + super().__init__(n_channels, d_model, n_tokens) + self.d_model = d_model + self.n_conv_layers = n_conv_layers + + # Calculate stride from input_length and n_tokens + # stride = (input_length / n_tokens)^(1 / n_conv_layers) + total_reduction = input_length / n_tokens + self.stride = int(math.ceil(total_reduction ** (1 / n_conv_layers))) + self.stride = max(2, min(self.stride, 5)) + + # Dynamically build channel progression: + # start at 64, double each layer, cap at d_model + intermediate = [min(64 * (2 ** i), d_model) for i in range(n_conv_layers - 1)] + self.channels = [n_channels] + intermediate + [d_model] + + # Build conv layers + self.conv_layers = nn.ModuleList([ + nn.Conv1d( + in_channels=self.channels[i], + out_channels=self.channels[i + 1], + kernel_size=kernel_size, + stride=self.stride, + padding=kernel_size // 2 + ) + for i in range(n_conv_layers) + ]) + + self.norms = nn.ModuleList([ + nn.InstanceNorm1d(self.channels[i + 1]) for i in range(n_conv_layers) + ]) + + self.adaptive_pool = nn.AdaptiveAvgPool1d(n_tokens) + self.activation = nn.GELU() + self.norm = nn.LayerNorm(d_model) def forward(self, x): - return self.proj(self.pool(self.conv_layers(x))) + """ + Encode time-series into tokens. + + Parameters + ---------- + x : torch.Tensor + Input time-series of shape [batch, n_channels, input_length] + + Returns + ------- + torch.Tensor + Encoded tokens of shape [batch, n_output_tokens, d_model] + """ + for conv, norm in zip(self.conv_layers, self.norms): + x = conv(x) # [B, channels[i+1], T'] + x = norm(x) + x = self.activation(x) + + x = self.adaptive_pool(x) # [B, d_model, n_output_tokens] + x = x.transpose(1, 2) # [B, n_output_tokens, d_model] + + return x class FastTimeSeriesBaselineDecoder(ModalityDecoder): + """ + Mirrors FastTimeSeriesEncoder for pre-training via masked autoencoding. + Reconstructs the original input time-series from encoder tokens. - def __init__(self, in_features=64, out_channels=1, target_length=5000, hidden_dim=128): - super().__init__(in_features, out_channels) - self.target_length = target_length - self.hidden_dim = hidden_dim - self.proj = nn.Sequential( - nn.Linear(in_features, hidden_dim * 2), - nn.ReLU(), - nn.Unflatten(1, (hidden_dim * 2, 1)), - ) - self.deconv_layers = nn.Sequential( - nn.ConvTranspose1d( - hidden_dim * 2, - hidden_dim * 2, - kernel_size=3, - stride=2, - padding=1, - output_padding=1, - ), - nn.GELU(), - nn.ConvTranspose1d( - hidden_dim * 2, - hidden_dim, - kernel_size=3, - stride=2, - padding=1, - output_padding=1, - ), - nn.GELU(), - nn.ConvTranspose1d( - hidden_dim, 64, kernel_size=5, stride=3, padding=1, output_padding=2 - ), - nn.GELU(), + Parameters + ---------- + n_channels : int, optional + Number of output channels (e.g., 6 for filterscopes), by default 6 + input_length : int, optional + Length of original input to reconstruct (e.g., 5000 for 500ms @ 10kHz), + by default 5000 + d_model : int, optional + Model dimension from encoder, by default 512 + n_input_tokens : int, optional + Number of input tokens from encoder, by default 100 + n_deconv_layers : int, optional + Number of deconvolutional layers (should match encoder), by default 4 + kernel_size : int, optional + Kernel size for transposed convolutions, by default 15 + + Attributes + ---------- + stride : int + Calculated stride for transposed convolutions + channels : list of int + Channel sizes at each layer, dynamically computed (reversed from encoder) + deconv_layers : nn.ModuleList + List of 1D transposed convolutional layers + adaptive_pool : nn.AdaptiveAvgPool1d + Adaptive pooling layer to ensure exact output length + """ + + def __init__( + self, + n_channels: int = 6, + input_length: int = 5000, + d_model: int = 512, + n_tokens: int = 100, + n_deconv_layers: int = 4, + kernel_size: int = 3, + ): + super().__init__(n_channels, n_tokens) + self.d_model = d_model + self.n_deconv_layers = n_deconv_layers + + # Mirror encoder stride calculation + total_expansion = input_length / n_tokens + self.stride = int(math.ceil(total_expansion ** (1 / n_deconv_layers))) + self.stride = max(2, min(self.stride, 5)) + + # Mirror encoder channel progression (reversed) + intermediate = [min(64 * (2 ** i), d_model) for i in range(n_deconv_layers - 1)] + self.channels = [d_model] + list(reversed(intermediate)) + [n_channels] + + # Build deconv layers + self.deconv_layers = nn.ModuleList([ nn.ConvTranspose1d( - 64, out_channels, kernel_size=10, stride=5, padding=2, output_padding=4 - ), - ) - self.resample = nn.AdaptiveAvgPool1d(target_length) + in_channels=self.channels[i], + out_channels=self.channels[i + 1], + kernel_size=kernel_size, + stride=self.stride, + padding=kernel_size // 2, + output_padding=self.stride - 1 + ) + for i in range(n_deconv_layers) + ]) + + self.adaptive_pool = nn.AdaptiveAvgPool1d(input_length) + self.activation = nn.GELU() + + def forward(self, x, output_shape=None): + """ + Decode tokens back to original time-series (pre-training only). + + Parameters + ---------- + x : torch.Tensor + Input tokens of shape [batch, n_input_tokens, d_model] + + Returns + ------- + torch.Tensor + Reconstructed time-series of shape [batch, n_channels, input_length] + """ + x = x.transpose(1, 2) # [B, d_model, n_input_tokens] + + for i, deconv in enumerate(self.deconv_layers): + x = deconv(x) + if i < len(self.deconv_layers) - 1: + x = self.activation(x) - def forward(self, z, output_shape=None): - return self.resample(self.deconv_layers(self.proj(z))) + x = self.adaptive_pool(x) # [B, n_channels, input_length] + + return x class FastTimeSeriesBaselineAutoEncoder(nn.Module): + """Combines TimeSeriesEncoder and TimeSeriesDecoder into an autoencoder model.""" - def __init__(self, n_channels, d_model=64, n_tokens=None): + def __init__( + self, + n_channels: int = 6, + input_length: int = 5000, + d_model: int = 512, + n_tokens: int = 100, + n_layers: int = 4, + kernel_size: int = 3, + ): super().__init__() - self.encoder = FastTimeSeriesBaselineEncoder(in_channels=n_channels, out_features=d_model) - self.decoder = FastTimeSeriesBaselineDecoder(in_features=d_model, out_channels=n_channels, target_length=5000) + self.encoder = FastTimeSeriesBaselineEncoder( + n_channels=n_channels, + input_length=input_length, + d_model=d_model, + n_tokens=n_tokens, + n_conv_layers=n_layers, + kernel_size=kernel_size, + ) + self.decoder = FastTimeSeriesBaselineDecoder( + n_channels=n_channels, + input_length=input_length, + d_model=d_model, + n_tokens=n_tokens, + n_deconv_layers=n_layers, + kernel_size=kernel_size, + ) def forward(self, x): - target_length = x.shape[-1] - z = self.encoder(x) - out = self.decoder(z) - if out.shape[-1] != target_length: - out = F.adaptive_avg_pool1d(out, target_length) - return out + """ + Forward pass through the autoencoder. + + Parameters + ---------- + x : torch.Tensor + Input time-series of shape [batch, n_channels, input_length] + + Returns + ------- + torch.Tensor + Reconstructed time-series of shape [batch, n_channels, input_length] + """ + tokens = self.encoder(x) + recon = self.decoder(tokens) + return recon def create_fast_timeseries_test_signal( batch_size: int = 4, @@ -161,7 +314,7 @@ def create_fast_timeseries_test_signal( print("FastTimeSeriesBaselineEncoder / FastTimeSeriesBaselineDecoder") print("=" * 60) ts_enc = FastTimeSeriesBaselineEncoder( - in_channels=6, + n_channels=6, out_features=512, hidden_dim=128, ) diff --git a/src/tokamak_foundation_model/models/model_factory.py b/src/tokamak_foundation_model/models/model_factory.py new file mode 100644 index 0000000..8c66174 --- /dev/null +++ b/src/tokamak_foundation_model/models/model_factory.py @@ -0,0 +1,44 @@ +from tokamak_foundation_model.models.modality import ( + ActuatorBaselineAutoEncoder, + SlowTimeSeriesBaselineAutoEncoder, + FastTimeSeriesBaselineAutoEncoder, + SpatialProfileBaselineAutoEncoder, + SpectrogramBaselineAutoEncoder, + VideoBaselineAutoEncoder, +) + + +SIGNAL_MODEL_DEFAULTS = { + "gas": "actuator", + "ech": "actuator", + "pin": "actuator", + "tin": "actuator", + "d_alpha": "fast_time_series", + "mse": "profile", + "ts_core_density": "profile", + "mhr": "spectrogram", + "ece": "spectrogram", + "co2": "spectrogram", + "bolo": "video", + "irtv": "video", + "tangtv": "video", +} + +MODEL_REGISTRY = { + "actuator": ActuatorBaselineAutoEncoder, + "fast_time_series": FastTimeSeriesBaselineAutoEncoder, + "slow_time_series": SlowTimeSeriesBaselineAutoEncoder, + "profile": SpatialProfileBaselineAutoEncoder, + "spectrogram": SpectrogramBaselineAutoEncoder, + "video": VideoBaselineAutoEncoder, +} + +def build_model(model_name, n_channels, d_model, n_tokens): + """Build the appropriate autoencoder. + + All autoencoders share the same interface: (n_channels, d_model, n_tokens). + """ + cls = MODEL_REGISTRY[model_name] + kwargs = dict(n_channels=n_channels, d_model=d_model) + if n_tokens is not None: kwargs["n_tokens"] = n_tokens + return cls(**kwargs) From 54f5e890737efd1591a9bc075ad5cb63f53bdeaa Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Mon, 16 Feb 2026 14:44:12 -0500 Subject: [PATCH 07/30] Lots of bugfixes in the dataset, trainer, and models. The basic encoders are now all working. Examples are in scripts. --- scripts/actuator_reconstruction.py | 16 +- scripts/fast_time_series_reconstruction.py | 218 +++++++++++---- scripts/profile_reconstruction.py | 250 +++++++++++++----- scripts/spectrogram_reconstruction.py | 190 +++++++++++++ .../data/data_loader.py | 17 +- .../models/modality/profile_baseline.py | 47 ++-- .../models/model_factory.py | 25 +- .../trainer/trainer.py | 25 +- 8 files changed, 628 insertions(+), 160 deletions(-) create mode 100644 scripts/spectrogram_reconstruction.py diff --git a/scripts/actuator_reconstruction.py b/scripts/actuator_reconstruction.py index 0af3da8..3b7da8c 100644 --- a/scripts/actuator_reconstruction.py +++ b/scripts/actuator_reconstruction.py @@ -28,7 +28,7 @@ def main(): parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") parser.add_argument( "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - default="pin", + default="gas", help="Signal name to train on" ) parser.add_argument( @@ -70,10 +70,10 @@ def main(): "--epochs", type=int, default=50, help="Number of training epochs" ) parser.add_argument( - "--lr", type=float, default=1e-3, help="Learning rate" + "--lr", type=float, default=5e-3, help="Learning rate" ) parser.add_argument( - "--weight_decay", type=float, default=0.05, help="AdamW weight decay" + "--weight_decay", type=float, default=1e-3, help="AdamW weight decay" ) parser.add_argument( "--warmup_epochs", type=int, default=5, @@ -111,7 +111,7 @@ def main(): logger.info(f"Signal: {signal_name}, Model: {model_name}") ### Dataset Setup ### - hdf5_files = sorted(data_dir.glob("*.h5")) + hdf5_files = sorted(data_dir.glob("*_processed.h5")) stats = torch.load(statistics_path) datasets_processed = [ @@ -144,6 +144,13 @@ def main(): model.parameters(), lr=args.lr, ) + + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=args.epochs, + eta_min=args.min_lr + ) + # loss_fn = nn.L1Loss() loss_fn = nn.MSELoss() @@ -165,6 +172,7 @@ def main(): checkpoint_path=checkpoint_path, model=model, optimizer=optimizer, + # lr_scheduler=lr_scheduler, loss_fn=loss_fn, device=device, drawer=drawer, diff --git a/scripts/fast_time_series_reconstruction.py b/scripts/fast_time_series_reconstruction.py index 6fd16fd..26df2d9 100644 --- a/scripts/fast_time_series_reconstruction.py +++ b/scripts/fast_time_series_reconstruction.py @@ -1,63 +1,181 @@ from pathlib import Path +import argparse +import logging + import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import ConcatDataset, DataLoader from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn -from tokamak_foundation_model.models.modality.fast_time_series_baseline import ( - TimeSeriesAutoencoder) +from tokamak_foundation_model.data.utils import worker_init_fn from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) +from tokamak_foundation_model.utils import DefaultDrawer -def worker_init_fn(worker_id): - """Each worker needs to open its own file handle.""" - worker_info = torch.utils.data.get_worker_info() - if worker_info is not None: - dataset = worker_info.dataset - # Force re-open file for this worker - if hasattr(dataset, 'datasets'): # ConcatDataset - for ds in dataset.datasets: - ds.h5_file = None - ds._open_hdf5() - else: - dataset.h5_file = None - dataset._open_hdf5() - - -hdf5_files = sorted( - Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") -) -stats = torch.load( - Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") -) - -datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=["d_alpha", ], - target_signals=["d_alpha", ], - prediction_mode=False, - ) - for f in hdf5_files -] - -concatenated_dataset = ConcatDataset(datasets_processed) - -dataloader = DataLoader( - concatenated_dataset, - batch_size=8, - shuffle=False, - collate_fn=collate_fn, - worker_init_fn=worker_init_fn - ) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = TimeSeriesAutoencoder() -model = model.to(device) -loss_fn = nn.MSELoss() -optimizer = optim.AdamW(model.parameters(), lr=0.005) -trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50) -trainer.train(dataloader, val_dataloader=dataloader, modality_key="d_alpha") +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser.add_argument( + "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + default="d_alpha", + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--hop_length", type=int, default=256, help="Hop length for STFT.", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default="fast_time_series", + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=512, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=140, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=4, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=50, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=5e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.05, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + signal_name = args.signal + model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Signal: {signal_name}, Model: {model_name}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*_processed.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=False, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + # Not sure if this is elegant + sample_data = next(iter(concatenated_dataset))[signal_name] + n_channels = sample_data.shape[0] + logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") + + ### Model Setup ### + model = build_model(model_name, n_channels, args.d_model, args.n_tokens).to(device) + + n_params = sum(p.numel() for p in model.parameters()) + logger.info(f"Model parameters: {n_params:,}") + + optimizer = optim.AdamW( + model.parameters(), + lr=args.lr, + ) + loss_fn = nn.L1Loss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = UnimodalTrainer( + epochs=args.epochs, + checkpoint_path=checkpoint_path, + model=model, + optimizer=optimizer, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=checkpoint_path) + + trainer.train(dataloader, modality_key=signal_name) + + +if __name__ == "__main__": + main() diff --git a/scripts/profile_reconstruction.py b/scripts/profile_reconstruction.py index 6377309..b6eff47 100644 --- a/scripts/profile_reconstruction.py +++ b/scripts/profile_reconstruction.py @@ -1,80 +1,194 @@ from pathlib import Path +import argparse +import logging + import torch import torch.nn as nn import torch.optim as optim from torch.utils.data import ConcatDataset, DataLoader from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn -from tokamak_foundation_model.models.modality.profile_baseline import ( - SpatialProfileEncoder, SpatialProfileDecoder) +from tokamak_foundation_model.data.utils import worker_init_fn from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) + +from tokamak_foundation_model.utils import DefaultDrawer -class DummyModel(torch.nn.Module): - def __init__(self): - super(DummyModel, self).__init__() - self.encoder = SpatialProfileEncoder( - kernel_size=3, n_spatial_points=44, n_time_points=50, d_model=512, - n_output_tokens=100) - self.decoder = SpatialProfileDecoder( - kernel_size=3, n_spatial_points=44, n_time_points=50, d_model=512, - n_input_tokens=100) - - def forward(self, x): - x_encoded = self.encoder(x) - return self.decoder(x_encoded) - - -def worker_init_fn(worker_id): - """Each worker needs to open its own file handle.""" - worker_info = torch.utils.data.get_worker_info() - if worker_info is not None: - dataset = worker_info.dataset - # Force re-open file for this worker - if hasattr(dataset, 'datasets'): # ConcatDataset - for ds in dataset.datasets: - ds.h5_file = None - ds._open_hdf5() - else: - dataset.h5_file = None - dataset._open_hdf5() - - -model = DummyModel() - - -hdf5_files = sorted( - Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/").glob("*_processed.h5") -) -stats = torch.load( - Path("C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt") -) - -datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=["ts_core_density", ], - target_signals=["ts_core_density", ], - prediction_mode=False, - ) - for f in hdf5_files -] - -concatenated_dataset = ConcatDataset(datasets_processed) - -dataloader = DataLoader( - concatenated_dataset, - batch_size=8, - shuffle=False, - collate_fn=collate_fn, - worker_init_fn=worker_init_fn - ) - -optimizer = optim.AdamW(model.parameters(), lr=0.005) -loss_fn = nn.L1Loss() # Be careful device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model = model.to(device) -trainer = UnimodalTrainer(model, optimizer, loss_fn, device=device, epochs=50) -trainer.train(dataloader, val_dataloader=dataloader, modality_key="ts_core_density") +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser.add_argument( + "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + default="ts_core_density", + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--hop_length", type=int, default=256, help="Hop length for STFT.", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default="profile", + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=512, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=140, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=4, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=50, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=5e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.01, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + signal_name = args.signal + model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Signal: {signal_name}, Model: {model_name}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*_processed.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=False, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + # Not sure if this is elegant + sample_data = next(iter(concatenated_dataset))[signal_name] + logger.info(f"Sample data shape: {sample_data.shape}") + n_spatial_points = sample_data.shape[0] + n_time_points = sample_data.shape[1] + logger.info(f"n_spatial_points: {n_spatial_points}, n_time_points: {n_time_points}") + ### Model Setup ### + model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, + n_channels=1, n_spatial_points=n_spatial_points, + n_time_points=n_time_points, kernel_size=3) + + model = model.to(device) + + n_params = sum(p.numel() for p in model.parameters()) + logger.info(f"Model parameters: {n_params:,}") + + optimizer = optim.AdamW( + model.parameters(), + lr=args.lr, + ) + + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=args.epochs, + eta_min=args.min_lr + ) + + loss_fn = nn.L1Loss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = UnimodalTrainer( + epochs=args.epochs, + checkpoint_path=checkpoint_path, + model=model, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=checkpoint_path) + + trainer.train(dataloader, modality_key=signal_name) + + +if __name__ == "__main__": + main() diff --git a/scripts/spectrogram_reconstruction.py b/scripts/spectrogram_reconstruction.py new file mode 100644 index 0000000..597443b --- /dev/null +++ b/scripts/spectrogram_reconstruction.py @@ -0,0 +1,190 @@ +from pathlib import Path +import argparse +import logging + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.data.utils import worker_init_fn +from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) + +from tokamak_foundation_model.utils import DefaultDrawer + + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser.add_argument( + "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + default="co2", + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--hop_length", type=int, default=256, help="Hop length for STFT.", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default="actuator", + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=512, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=140, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=1, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=50, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=5e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=1e-3, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + signal_name = args.signal + model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Signal: {signal_name}, Model: {model_name}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*_processed.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=False, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + # Not sure if this is elegant + sample_data = next(iter(concatenated_dataset))[signal_name] + n_channels = sample_data.shape[0] + logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") + + ### Model Setup ### + model = build_model(model_name, n_channels, args.d_model, args.n_tokens).to(device) + + n_params = sum(p.numel() for p in model.parameters()) + logger.info(f"Model parameters: {n_params:,}") + + optimizer = optim.AdamW( + model.parameters(), + lr=args.lr, + ) + + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=args.epochs, + eta_min=args.min_lr + ) + + # loss_fn = nn.L1Loss() + loss_fn = nn.MSELoss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = UnimodalTrainer( + epochs=args.epochs, + checkpoint_path=checkpoint_path, + model=model, + optimizer=optimizer, + # lr_scheduler=lr_scheduler, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=checkpoint_path) + + trainer.train(dataloader, modality_key=signal_name) + + +if __name__ == "__main__": + main() diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index cfa697e..e35d803 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -49,6 +49,8 @@ def compute_preprocessing_stats( all_values = torch.cat(values, dim=1) # (channels, time) elif values[0].ndim == 3: all_values = torch.cat(values, dim=2) # (channels, freq_bins, time) + else: + raise ValueError(f"Invalid tensor shape: {values[0].shape}") # Compute per-channel statistics # Reduce over all dimensions except channel dimension (dim=1) @@ -151,7 +153,7 @@ class TokamakH5Dataset(Dataset): 4, 500e3, apply_stft=True, - preprocess=PreprocessConfig(method="standardize"), + preprocess=PreprocessConfig(method="log_standardize"), ), SignalConfig( "d_alpha", @@ -159,7 +161,7 @@ class TokamakH5Dataset(Dataset): 6, 10e3, apply_stft=False, - preprocess=PreprocessConfig(method="none"), + preprocess=PreprocessConfig(method="standardize"), ), SignalConfig( "gas", @@ -337,7 +339,7 @@ def _apply_preprocessing( return (tensor - min_val) / (max_val - min_val + config.eps) elif config.method == "log_standardize": - tensor_log = torch.log(tensor + 1) + tensor_log = torch.log10(tensor + 1) if config.mean is None or config.std is None: print("Warning: log_standardize requested but no statistics provided") @@ -355,6 +357,10 @@ def _apply_preprocessing( return (tensor_log - mean) / (std + config.eps) + elif config.method == "log": + tensor_log = torch.log10(tensor + 1) + return tensor_log + return tensor def _compute_duration_from_handle(self, f: h5py.File) -> float: @@ -415,12 +421,11 @@ def _load_signal_raw( t1 = xdata_ds[-1] / 1000.0 n_samples = xdata_ds.shape[0] - duration_s = t_end - t_start - fs_raw = (n_samples - 1) / (t1 - t0) + duration_s = t_end - t_start ydata = np.zeros( - (max(1, round(duration_s * fs_raw)), config.num_channels), dtype=np.float32 + (round(duration_s * fs_raw), config.num_channels), dtype=np.float32 ) start_idx = max(0, int((t_start - t0) * fs_raw)) diff --git a/src/tokamak_foundation_model/models/modality/profile_baseline.py b/src/tokamak_foundation_model/models/modality/profile_baseline.py index fbf9f78..c79da54 100644 --- a/src/tokamak_foundation_model/models/modality/profile_baseline.py +++ b/src/tokamak_foundation_model/models/modality/profile_baseline.py @@ -82,7 +82,7 @@ def __init__(self, self.n_tokens = n_tokens self.activation = nn.GELU() - self.adaptive_pool = nn.AdaptiveAvgPool1d(n_tokens) + self.adaptive_pool = nn.AdaptiveAvgPool1d(n_time_points) # Mirror temporal conv self.temporal_deconv = nn.ConvTranspose1d( @@ -103,35 +103,44 @@ def __init__(self, nn.Linear(128, n_spatial_points) ) - def forward(self, z, output_shape=None): - B, D, T = z.shape + def forward(self, x, output_shape=None): + B = x.shape[0] # Upsample temporal dimension - z = z.transpose(1, 2) # [B, d_model, n_input_tokens] - z = self.activation(self.temporal_deconv(z)) # [B, d_model, T'] - z = self.adaptive_pool(z) # [B, d_model, n_time] + x = x.transpose(1, 2) # [B, d_model, n_input_tokens] + x = self.activation(self.temporal_deconv(x)) # [B, d_model, T'] + x = self.adaptive_pool(x) # [B, d_model, n_time] # Decode spatial structure at each time step independently - z = z.transpose(1, 2) # [B, n_time, d_model] - T = z.shape[1] - z = z.reshape(B * T, self.d_model) # [B*T, d_model] - z = self.spatial_decoder(z) # [B*n_time, n_spatial] - z = z.reshape(B, T, self.n_spatial_points) # [B, n_time, n_spatial] - z = z.transpose(1, 2) # [B, n_spatial, n_time] + x = x.transpose(1, 2) # [B, n_time, d_model] + T = x.shape[1] + x = x.reshape(B * T, self.d_model) # [B*T, d_model] + x = self.spatial_decoder(x) # [B*n_time, n_spatial] + x = x.reshape(B, T, self.n_spatial_points) # [B, n_time, n_spatial] + x = x.transpose(1, 2) # [B, n_spatial, n_time] - return z + return x class SpatialProfileBaselineAutoEncoder(ModalityAutoEncoder): - def __init__(self, - n_channels: int, - d_model: int = 64, - n_tokens: int = 0, + def __init__( + self, + n_channels: int, + d_model: int = 64, + n_tokens: int = 0, + n_spatial_points: int = 50, + n_time_points: int = 50, + kernel_size: int = 3, ): super().__init__(n_channels, d_model, n_tokens) - self.encoder = SpatialProfileBaselineEncoder(n_channels, d_model, n_tokens) - self.decoder = SpatialProfileBaselineDecoder(n_channels, d_model, n_tokens) + + self.encoder = SpatialProfileBaselineEncoder(n_channels, d_model, n_tokens, + n_spatial_points, n_time_points, + kernel_size) + self.decoder = SpatialProfileBaselineDecoder(n_channels, d_model, n_tokens, + n_spatial_points, n_time_points, + kernel_size) def forward(self, x): n_time = x.shape[-1] diff --git a/src/tokamak_foundation_model/models/model_factory.py b/src/tokamak_foundation_model/models/model_factory.py index 8c66174..4570451 100644 --- a/src/tokamak_foundation_model/models/model_factory.py +++ b/src/tokamak_foundation_model/models/model_factory.py @@ -1,3 +1,6 @@ +from torch import nn +from typing import Optional + from tokamak_foundation_model.models.modality import ( ActuatorBaselineAutoEncoder, SlowTimeSeriesBaselineAutoEncoder, @@ -33,12 +36,28 @@ "video": VideoBaselineAutoEncoder, } -def build_model(model_name, n_channels, d_model, n_tokens): +def build_model( + model_name, + d_model: Optional[int], + n_tokens: Optional[int], + n_channels: Optional[int], + **kwargs +) -> nn.Module: """Build the appropriate autoencoder. All autoencoders share the same interface: (n_channels, d_model, n_tokens). """ cls = MODEL_REGISTRY[model_name] - kwargs = dict(n_channels=n_channels, d_model=d_model) - if n_tokens is not None: kwargs["n_tokens"] = n_tokens + if d_model is None and "d_model" not in kwargs: + kwargs["d_model"] = 512 # default model dimension + else: + kwargs["d_model"] = d_model + if n_tokens is None and "n_tokens" not in kwargs: + kwargs["n_tokens"] = 20 + else: + kwargs["n_tokens"] = n_tokens + if n_channels is None and "n_channels" not in kwargs: + kwargs["n_channels"] = 1 + else: + kwargs["n_channels"] = n_channels return cls(**kwargs) diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index a95bb2f..4806f91 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -89,18 +89,21 @@ def load_checkpoint(self, checkpoint_path=None): class UnimodalTrainer: - def __init__(self, - model: nn.Module, - optimizer: optim.Optimizer, - loss_fn: nn.Module, - device: torch.device, - epochs: int, - log_interval: int | None = None, - drawer: object | None = None, - checkpoint_path: str | Path = "checkpoint.pth", - ): + def __init__( + self, + model: nn.Module, + optimizer: optim.Optimizer, + loss_fn: nn.Module, + device: torch.device, + epochs: int, + lr_scheduler: optim.lr_scheduler.LRScheduler | None = None, + log_interval: int | None = None, + drawer: object | None = None, + checkpoint_path: str | Path = "checkpoint.pth", + ): self.model = model self.optimizer = optimizer + self.lr_scheduler = lr_scheduler self.loss_fn = loss_fn self.device = device self.epochs = epochs @@ -187,6 +190,8 @@ def train(self, torch.save(self.model.state_dict(), self.best_checkpoint_path) logger.info(f" Best validation loss: {best_val_loss:.4f}, best model checkpoint saved!") + self.lr_scheduler.step() + # Logging if self.log_interval is not None: if epoch % self.log_interval == 0: From 3195edc3439697bb74cc2bf6c930de42c4709906 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:19:56 -0500 Subject: [PATCH 08/30] Extended checkpointing - the trainer stores now: - Model - Optimizer state - Scheduler state - Current loss - Current epoch For the sake of continual training. --- src/tokamak_foundation_model/trainer/trainer.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index 4806f91..048fc3f 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -175,12 +175,19 @@ def train(self, for epoch in range(self.epochs): self._current_epoch = epoch - logger.info(f"Epoch {epoch+1}/{self.epochs}") + logger.info(f"Epoch {epoch + 1}/{self.epochs}") train_loss = self._train_epoch(train_dataloader, modality_key) logger.info(f" Training Loss: {train_loss:.4f}") - torch.save(self.model.state_dict(), self.checkpoint_path) - + torch.save( + {"model": self.model, + "optimizer_state_dict": self.optimizer.state_dict(), + "scheduler_state_dict": self.lr_scheduler.state_dict(), + "epoch": epoch, + "loss": train_loss, + }, + self.checkpoint_path) + # Validation if val_dataloader: val_loss = self._validate_epoch(val_dataloader, modality_key) @@ -188,7 +195,8 @@ def train(self, if val_loss < best_val_loss: best_val_loss = val_loss torch.save(self.model.state_dict(), self.best_checkpoint_path) - logger.info(f" Best validation loss: {best_val_loss:.4f}, best model checkpoint saved!") + logger.info(f" Best validation loss: {best_val_loss:.4f}, " + f"best model checkpoint saved!") self.lr_scheduler.step() From a0edc4d2d2a14a5963a974d902dafef4457d2130 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:20:51 -0500 Subject: [PATCH 09/30] Extended checkpointing - the trainer stores now: - Model - Optimizer state - Scheduler state - Current loss - Current epoch For the sake of continual training. --- .../trainer/trainer.py | 128 ++++++++++-------- 1 file changed, 74 insertions(+), 54 deletions(-) diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index 048fc3f..de2ac62 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -11,14 +11,16 @@ logger = logging.getLogger(__name__) + class MultimodalTrainer: - def __init__(self, - model: nn.Module, - optimizer: optim.Optimizer, - loss_fn: nn.Module, - device: torch.device, + def __init__( + self, + model: nn.Module, + optimizer: optim.Optimizer, + loss_fn: nn.Module, + device: torch.device, epochs: int, - checkpoint_path: str | Path = "checkpoint.pth" + checkpoint_path: str | Path = "checkpoint.pth", ): self.model = model self.optimizer = optimizer @@ -31,10 +33,16 @@ def _train_epoch(self, dataloader: DataLoader): self.model.train() total_loss = 0 for batch_idx, batch in enumerate(dataloader): - inputs = batch['inputs'] - targets = batch['targets'] - inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()} - targets = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in targets.items()} + inputs = batch["inputs"] + targets = batch["targets"] + inputs = { + k: v.to(self.device) if isinstance(v, torch.Tensor) else v + for k, v in inputs.items() + } + targets = { + k: v.to(self.device) if isinstance(v, torch.Tensor) else v + for k, v in targets.items() + } self.optimizer.zero_grad() outputs = self.model(inputs) @@ -52,8 +60,12 @@ def _validate_epoch(self, dataloader: DataLoader): total_loss = 0 with torch.no_grad(): for batch_idx, batch in enumerate(dataloader): - inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in batch.items() if k != 'target'} - targets = batch['target'].to(self.device).float().unsqueeze(1) + inputs = { + k: v.to(self.device) if isinstance(v, torch.Tensor) else v + for k, v in batch.items() + if k != "target" + } + targets = batch["target"].to(self.device).float().unsqueeze(1) outputs = self.model(inputs) loss = self.loss_fn(outputs, targets) @@ -61,9 +73,9 @@ def _validate_epoch(self, dataloader: DataLoader): return total_loss / len(dataloader) def train(self, train_dataloader: DataLoader, val_dataloader: DataLoader = None): - best_val_loss = float('inf') + best_val_loss = float("inf") for epoch in range(self.epochs): - print(f"Epoch {epoch+1}/{self.epochs}") + print(f"Epoch {epoch + 1}/{self.epochs}") train_loss = self._train_epoch(train_dataloader) print(f" Training Loss: {train_loss:.4f}") @@ -90,16 +102,16 @@ def load_checkpoint(self, checkpoint_path=None): class UnimodalTrainer: def __init__( - self, - model: nn.Module, - optimizer: optim.Optimizer, - loss_fn: nn.Module, - device: torch.device, - epochs: int, - lr_scheduler: optim.lr_scheduler.LRScheduler | None = None, - log_interval: int | None = None, - drawer: object | None = None, - checkpoint_path: str | Path = "checkpoint.pth", + self, + model: nn.Module, + optimizer: optim.Optimizer, + loss_fn: nn.Module, + device: torch.device, + epochs: int, + lr_scheduler: optim.lr_scheduler.LRScheduler | None = None, + log_interval: int | None = None, + drawer: object | None = None, + checkpoint_path: str | Path = "checkpoint.pth", ): self.model = model self.optimizer = optimizer @@ -114,23 +126,26 @@ def __init__( p = Path(checkpoint_path) self.best_checkpoint_path = p.with_name(p.stem + "_best" + p.suffix) - def _log_epoch(self, - epoch: int, - train_loss: float, + def _log_epoch( + self, + epoch: int, + train_loss: float, val_loss: float = 0, - ): - logger.info(f"Epoch {epoch+1}/{self.epochs}," + - f"Training Loss: {train_loss:.4f}," + - f"Validation Loss: {val_loss:.4f}" - ) - + ): + logger.info( + f"Epoch {epoch + 1}/{self.epochs}," + + f"Training Loss: {train_loss:.4f}," + + f"Validation Loss: {val_loss:.4f}" + ) + if self.drawer: self.drawer(self.model, epoch, train_loss, val_loss) - def _train_epoch(self, - dataloader: DataLoader, + def _train_epoch( + self, + dataloader: DataLoader, modality_key: str, - ): + ): self.model.train() total_loss = 0 for batch_idx, batch in enumerate(dataloader): @@ -143,10 +158,11 @@ def _train_epoch(self, total_loss += loss.item() return total_loss / len(dataloader) - def _validate_epoch(self, - dataloader: DataLoader, + def _validate_epoch( + self, + dataloader: DataLoader, modality_key: str, - ): + ): self.model.eval() total_loss = 0 with torch.no_grad(): @@ -157,16 +173,16 @@ def _validate_epoch(self, total_loss += loss.item() return total_loss / len(dataloader) - def train(self, - train_dataloader: DataLoader, + def train( + self, + train_dataloader: DataLoader, val_dataloader: DataLoader = None, - modality_key: str = 'dalpha', - ): - + modality_key: str = "dalpha", + ): # Setup Training Loop self._current_epoch = 0 train_loss, val_loss = 0, 0 - best_val_loss = float('inf') + best_val_loss = float("inf") if self.drawer: self.drawing_path = Path(self.checkpoint_path).parent / "plots" self.drawer.setup(train_dataloader, self.drawing_path, modality_key) @@ -180,13 +196,15 @@ def train(self, logger.info(f" Training Loss: {train_loss:.4f}") torch.save( - {"model": self.model, - "optimizer_state_dict": self.optimizer.state_dict(), - "scheduler_state_dict": self.lr_scheduler.state_dict(), - "epoch": epoch, - "loss": train_loss, - }, - self.checkpoint_path) + { + "model": self.model, + "optimizer_state_dict": self.optimizer.state_dict(), + "scheduler_state_dict": self.lr_scheduler.state_dict(), + "epoch": epoch, + "loss": train_loss, + }, + self.checkpoint_path, + ) # Validation if val_dataloader: @@ -195,8 +213,10 @@ def train(self, if val_loss < best_val_loss: best_val_loss = val_loss torch.save(self.model.state_dict(), self.best_checkpoint_path) - logger.info(f" Best validation loss: {best_val_loss:.4f}, " - f"best model checkpoint saved!") + logger.info( + f" Best validation loss: {best_val_loss:.4f}, " + f"best model checkpoint saved!" + ) self.lr_scheduler.step() From 4d019845dba1c8e04db65ebbb848da50d0792cd4 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Mon, 16 Feb 2026 16:39:18 -0500 Subject: [PATCH 10/30] Adapted the other reconstruction scripts to match the new API. --- scripts/actuator_reconstruction.py | 7 ++++--- scripts/fast_time_series_reconstruction.py | 11 ++++++++++- scripts/profile_reconstruction.py | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/actuator_reconstruction.py b/scripts/actuator_reconstruction.py index 3b7da8c..a6147ba 100644 --- a/scripts/actuator_reconstruction.py +++ b/scripts/actuator_reconstruction.py @@ -28,7 +28,7 @@ def main(): parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") parser.add_argument( "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - default="gas", + default="pin", help="Signal name to train on" ) parser.add_argument( @@ -135,7 +135,8 @@ def main(): logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") ### Model Setup ### - model = build_model(model_name, n_channels, args.d_model, args.n_tokens).to(device) + model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, + n_channels=n_channels, kernel_size=3).to(device) n_params = sum(p.numel() for p in model.parameters()) logger.info(f"Model parameters: {n_params:,}") @@ -172,7 +173,7 @@ def main(): checkpoint_path=checkpoint_path, model=model, optimizer=optimizer, - # lr_scheduler=lr_scheduler, + lr_scheduler=lr_scheduler, loss_fn=loss_fn, device=device, drawer=drawer, diff --git a/scripts/fast_time_series_reconstruction.py b/scripts/fast_time_series_reconstruction.py index 26df2d9..808037d 100644 --- a/scripts/fast_time_series_reconstruction.py +++ b/scripts/fast_time_series_reconstruction.py @@ -135,7 +135,8 @@ def main(): logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") ### Model Setup ### - model = build_model(model_name, n_channels, args.d_model, args.n_tokens).to(device) + model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, + n_channels=n_channels, kernel_size=3).to(device) n_params = sum(p.numel() for p in model.parameters()) logger.info(f"Model parameters: {n_params:,}") @@ -144,6 +145,13 @@ def main(): model.parameters(), lr=args.lr, ) + + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=args.epochs, + eta_min=args.min_lr + ) + loss_fn = nn.L1Loss() dataloader = DataLoader( @@ -164,6 +172,7 @@ def main(): checkpoint_path=checkpoint_path, model=model, optimizer=optimizer, + lr_scheduler=lr_scheduler, loss_fn=loss_fn, device=device, drawer=drawer, diff --git a/scripts/profile_reconstruction.py b/scripts/profile_reconstruction.py index b6eff47..91500d9 100644 --- a/scripts/profile_reconstruction.py +++ b/scripts/profile_reconstruction.py @@ -28,7 +28,7 @@ def main(): parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") parser.add_argument( "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - default="ts_core_density", + default="mse", help="Signal name to train on" ) parser.add_argument( From 8b457c8663ffe19b4320976525841259296db4b9 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Mon, 16 Feb 2026 17:06:30 -0500 Subject: [PATCH 11/30] Bugfix in the dataset class. When splitting inputs and targets, I forgot to remove unused modalities. This follows the standard getitem function now. --- src/tokamak_foundation_model/data/data_loader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index e35d803..cd20489 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -708,6 +708,8 @@ def _getitem_prediction(self, idx): # For signals: split at input_frames for config in self.signal_configs: + if config.name not in signals_to_load: + continue signal = all_signals[config.name] if config.apply_stft: @@ -725,6 +727,8 @@ def _getitem_prediction(self, idx): # Movies: split along time dimension for movie_config in self.movie_configs: + if movie_config.name not in signals_to_load: + continue movie_name = movie_config.name movie_data = all_movies[movie_name] n_training_frames = round(self.chunk_duration_s * movie_config.target_fps) From 930fd2759a7a31304c5b4c21cca6cd394688aef1 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Mon, 16 Feb 2026 17:43:15 -0500 Subject: [PATCH 12/30] Prepared an option to preprocess movies. This has to be fully integrated!!! --- .../data/data_loader.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index cd20489..dd4ff53 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -74,18 +74,6 @@ def compute_preprocessing_stats( return stats -@dataclass -class MovieConfig: - """Configuration for a movie/video diagnostic.""" - - name: str # Key in output dict - hdf5_keys: list[str] # Possible HDF5 paths to search - channels: int # Color channels (e.g., 3 for RGB) - target_fps: int # Target frames per second after resampling - height: int # Frame height - width: int # Frame width - - @dataclass class PreprocessConfig: """Preprocessing configuration.""" @@ -114,6 +102,23 @@ def __post_init__(self): self.preprocess = PreprocessConfig() +@dataclass +class MovieConfig: + """Configuration for a movie/video diagnostic.""" + + name: str # Key in output dict + hdf5_keys: list[str] # Possible HDF5 paths to search + channels: int # Color channels (e.g., 3 for RGB) + target_fps: int # Target frames per second after resampling + height: int # Frame height + width: int # Frame width + preprocess: PreprocessConfig = None # Add preprocessing config + + def __post_init__(self): + if self.preprocess is None: + self.preprocess = PreprocessConfig() + + class TokamakH5Dataset(Dataset): """ Dataset for loading multi-modal tokamak data from HDF5 files. From 357ada7328514a73286ac18b5a6de682252482cf Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Tue, 17 Feb 2026 09:13:01 -0500 Subject: [PATCH 13/30] Added a baseline fusion transformer for latent space prediction. Quick fix for the data standardization. Invalid values have to be ignored. Fix in the function to create H5 files. bolo data does not have to be flipped anymore as the data is now stored in the correct format. --- ...train_multimodal_latent_space_predictor.py | 287 ++++++++++++++++++ .../data/data_loader.py | 14 +- .../data/dummy_data.py | 2 +- .../models/latent_feature_space/__init__.py | 0 .../baseline_fusion_transformer.py | 188 ++++++++++++ 5 files changed, 486 insertions(+), 5 deletions(-) create mode 100644 scripts/train_multimodal_latent_space_predictor.py create mode 100644 src/tokamak_foundation_model/models/latent_feature_space/__init__.py create mode 100644 src/tokamak_foundation_model/models/latent_feature_space/baseline_fusion_transformer.py diff --git a/scripts/train_multimodal_latent_space_predictor.py b/scripts/train_multimodal_latent_space_predictor.py new file mode 100644 index 0000000..b2b30bd --- /dev/null +++ b/scripts/train_multimodal_latent_space_predictor.py @@ -0,0 +1,287 @@ +from pathlib import Path +import argparse +import logging + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.data.utils import worker_init_fn +from tokamak_foundation_model.trainer.trainer import MultimodalTrainer +from tokamak_foundation_model.models.model_factory import SIGNAL_MODEL_DEFAULTS +from tokamak_foundation_model.models.latent_feature_space.baseline_fusion_transformer \ + import BaselineFusionTransformer # , BaselineForecastingDecoder +from tokamak_foundation_model.utils import DefaultDrawer + + +# Signals that are input-only (not predicted at output) +INPUT_ONLY_SIGNALS = [key for key, value in SIGNAL_MODEL_DEFAULTS.items() if value == + "actuator"] # Only diagnostic signals are currently predicted + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def load_frozen_encoder(checkpoint_path: Path, device: torch.device) -> nn.Module: + """ + Load pre-trained autoencoder from checkpoint and extract frozen encoder. + + Parameters + ---------- + checkpoint_path : Path + Path to the autoencoder checkpoint + device : torch.device + Device to load the model on + + Returns + ------- + nn.Module + Frozen encoder extracted from the autoencoder + """ + checkpoint = torch.load(checkpoint_path, weights_only=False, map_location=device) + logger.info( + f"Loaded checkpoint from {checkpoint_path}: " + f"epoch {checkpoint['epoch']}, loss {checkpoint['loss']:.4f}" + ) + model = checkpoint["model"] + encoder = model.encoder + + # Freeze all encoder parameters + for param in encoder.parameters(): + param.requires_grad = False + encoder.eval() + + return encoder + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser( + description="Train multimodal fusion transformer with forecasting decoders" + ) + parser.add_argument( + "--signals", required=False, nargs="+", + default=['d_alpha', 'mse', 'pin', 'tin', 'ts_core_density', 'irtv'], + choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + help="List of input signal names" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size" + ) + parser.add_argument( + "--hop_length", type=int, default=512, help="STFT hop length" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, default="preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", + help="Directory containing pre-trained autoencoder checkpoints " + "and saving fusion model checkpoints" + ) + parser.add_argument( + "--d_model", type=int, default=64, help="Model dimension" + ) + parser.add_argument( + "--n_heads", type=int, default=8, help="Number of attention heads" + ) + parser.add_argument( + "--n_layers", type=int, default=6, help="Number of transformer layers" + ) + parser.add_argument( + "--dropout", type=float, default=0.1, help="Dropout rate" + ) + parser.add_argument( + "--batch_size", type=int, default=2, help="Batch size" + ) + parser.add_argument( + "--num_workers", type=int, default=4, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=10, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=1e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.05, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, + help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + checkpoint_dir = Path(args.checkpoint_dir) + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + fusion_checkpoint_path = checkpoint_dir / "fusion" / "checkpoint.pth" + fusion_checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + ### Resolve input and output signals ### + input_signals = args.signals + output_signals = [s for s in input_signals if s not in INPUT_ONLY_SIGNALS] + + logger.info(f"Input signals: {input_signals}") + logger.info(f"Output signals: {output_signals}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*_processed.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=input_signals, + target_signals=output_signals, + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=True, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + ### Load frozen encoders ### + encoders = {} + for signal_name in input_signals: + model_name = SIGNAL_MODEL_DEFAULTS[signal_name] + ckpt_path = checkpoint_dir / f"{signal_name}_{model_name}" / "checkpoint.pth" + + if not ckpt_path.exists(): + raise FileNotFoundError( + f"Pre-trained checkpoint not found for signal '{signal_name}' " + f"at {ckpt_path}. Run unimodal pre-training first." + ) + + encoders[signal_name] = load_frozen_encoder(ckpt_path, device) + logger.info(f"Loaded frozen encoder for: {signal_name}") + + ### Infer token counts and output shapes from sample data ### + data = next(iter(concatenated_dataset)) + + # Total tokens across all modalities (for transformer max_tokens) + total_tokens = 0 + modality_token_counts = {} + for signal_name, encoder in encoders.items(): + with torch.no_grad(): + sample = data["inputs"][signal_name].unsqueeze(0).to(device) + tokens = encoder(sample) + modality_token_counts[signal_name] = tokens.shape[1] + total_tokens += tokens.shape[1] + logger.info( + f"Signal '{signal_name}': {tokens.shape[1]} tokens, " + f"shape {tokens.shape}" + ) + + # Output shapes for forecasting decoders + output_shapes = {} + for signal_name in output_signals: + output_shapes[signal_name] = tuple(data["targets"][signal_name].shape) + logger.info(f"Output '{signal_name}': shape {output_shapes[signal_name]}") + + ### Model Setup ### + fusion_transformer = BaselineFusionTransformer( + d_model=args.d_model, + n_heads=args.n_heads, + n_layers=args.n_layers, + dropout=args.dropout, + n_modalities=len(input_signals), + max_tokens=total_tokens, + ).to(device) + + """ + forecasting_decoders = nn.ModuleDict({ + signal_name: BaselineForecastingDecoder( + output_shape=output_shapes[signal_name], + d_model=args.d_model, + ).to(device) + for signal_name in output_signals + }) + """ + + n_params_transformer = sum( + p.numel() for p in fusion_transformer.parameters() + ) + """ + n_params_decoders = sum( + p.numel() for p in forecasting_decoders.parameters() + ) + """ + logger.info(f"Fusion transformer parameters: {n_params_transformer:,}") + """ + logger.info(f"Forecasting decoder parameters: {n_params_decoders:,}") + """ + # Only optimize transformer and forecasting decoders (encoders are frozen) + optimizer = optim.AdamW( + list(fusion_transformer.parameters()), # + list(forecasting_decoders.parameters()) + lr=args.lr, + weight_decay=args.weight_decay, + ) + + loss_fn = nn.L1Loss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = MultimodalTrainer( + epochs=args.epochs, + checkpoint_path=fusion_checkpoint_path, + encoders=encoders, + fusion_transformer=fusion_transformer, + forecasting_decoders=forecasting_decoders, + optimizer=optimizer, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and fusion_checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {fusion_checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=fusion_checkpoint_path) + + trainer.train(dataloader) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index dd4ff53..433cf8b 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -57,10 +57,16 @@ def compute_preprocessing_stats( dims_to_reduce = list(range(all_values.ndim)) dims_to_reduce.remove(0) # Keep channel dimension - mean = all_values.mean(dim=dims_to_reduce) - std = all_values.std(dim=dims_to_reduce) - min_val = all_values.min() - max_val = all_values.max() + valid_mask = ~torch.isnan(all_values) + + # For mean/std: use nanmean + manual std + mean = all_values.nanmean(dim=dims_to_reduce) + mean_expanded = mean.view(-1, *([1] * (all_values.ndim - 1))) + std = ((all_values - mean_expanded) ** 2).nanmean(dim=dims_to_reduce).sqrt() + + # For min/max: mask out NaNs with inf + min_val = all_values.nan_to_num(posinf=float("inf"), nan=float("inf")).min() + max_val = all_values.nan_to_num(neginf=float("-inf"), nan=float("-inf")).max() stats[config.name] = { "mean": mean, diff --git a/src/tokamak_foundation_model/data/dummy_data.py b/src/tokamak_foundation_model/data/dummy_data.py index 9a7af49..2453dbe 100644 --- a/src/tokamak_foundation_model/data/dummy_data.py +++ b/src/tokamak_foundation_model/data/dummy_data.py @@ -263,7 +263,7 @@ def create_single_sample_hdf5(): signal_group.create_dataset("ydata", data=tin[1]) signal_group = f.create_group("bolo") signal_group.create_dataset("xdata", data=bolo[0]) - signal_group.create_dataset("ydata", data=bolo[1].swapaxes(0, 2)) + signal_group.create_dataset("ydata", data=bolo[1]) signal_group = f.create_group("irtv") signal_group.create_dataset("xdata", data=irtv[0]) signal_group.create_dataset("ydata", data=irtv[1]) diff --git a/src/tokamak_foundation_model/models/latent_feature_space/__init__.py b/src/tokamak_foundation_model/models/latent_feature_space/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/tokamak_foundation_model/models/latent_feature_space/baseline_fusion_transformer.py b/src/tokamak_foundation_model/models/latent_feature_space/baseline_fusion_transformer.py new file mode 100644 index 0000000..abbca73 --- /dev/null +++ b/src/tokamak_foundation_model/models/latent_feature_space/baseline_fusion_transformer.py @@ -0,0 +1,188 @@ +import torch +import torch.nn as nn + +class BaselineFusionTransformer(nn.Module): + """ + Baseline transformer for joint latent feature fusion and prediction. + Concatenates tokens from all modalities and processes them with a + standard causal transformer. + + Parameters + ---------- + d_model : int, optional + Model dimension, by default 512 + n_heads : int, optional + Number of attention heads, by default 8 + n_layers : int, optional + Number of transformer layers, by default 6 + dropout : float, optional + Dropout rate, by default 0.1 + n_modalities : int, optional + Number of input modalities for learned modality embeddings, by default 5 + max_tokens : int, optional + Maximum total number of tokens across all modalities, by default 1024 + verbose : bool, optional + If True, print debug information during initialization, by default False + + Attributes + ---------- + modality_embeddings : nn.Embedding + Learned embedding added per modality to distinguish token sources + position_embeddings : nn.Embedding + Learned positional embeddings over token sequence + transformer : nn.TransformerEncoder + Stack of causal transformer encoder layers + norm : nn.LayerNorm + Final layer norm + """ + + def __init__( + self, + d_model: int = 512, + n_heads: int = 8, + n_layers: int = 6, + dropout: float = 0.1, + n_modalities: int = 5, + max_tokens: int = 1024, + verbose: bool = False + ): + super().__init__() + + self.d_model = d_model + self.n_heads = n_heads + self.n_layers = n_layers + self.n_modalities = n_modalities + self.max_tokens = max_tokens + self.verbose = verbose + + # Learned modality embeddings (one per modality) + self.modality_embeddings = nn.Embedding(n_modalities, d_model) + + # Learned positional embeddings over full token sequence + self.position_embeddings = nn.Embedding(max_tokens, d_model) + + # Standard transformer encoder layer with pre-LayerNorm + encoder_layer = nn.TransformerEncoderLayer( + d_model=d_model, + nhead=n_heads, + dim_feedforward=d_model * 4, + dropout=dropout, + activation='gelu', + batch_first=True, + norm_first=True # pre-LayerNorm (more stable) + ) + + self.transformer = nn.TransformerEncoder( + encoder_layer=encoder_layer, + num_layers=n_layers, + norm=nn.LayerNorm(d_model) + ) + + if self.verbose: + print(f"BaselineFusionTransformer:") + print(f" d_model: {d_model}") + print(f" n_heads: {n_heads}") + print(f" n_layers: {n_layers}") + print(f" n_modalities: {n_modalities}") + print(f" max_tokens: {max_tokens}") + + def _causal_mask(self, n_tokens: int, device: torch.device) -> torch.Tensor: + """ + Generate causal attention mask. + + Parameters + ---------- + n_tokens : int + Number of tokens in the sequence + device : torch.device + Device to create mask on + + Returns + ------- + torch.Tensor + Causal mask of shape [n_tokens, n_tokens] where future + positions are masked with -inf + """ + return torch.triu( + torch.full((n_tokens, n_tokens), float('-inf'), device=device), + diagonal=1 + ) + + def forward(self, token_list: list[tuple[torch.Tensor, int]]) -> torch.Tensor: + """ + Fuse and process tokens from all modalities. + + Parameters + ---------- + token_list : list of tuple of (torch.Tensor, int) + Each entry is (tokens, modality_id) where: + - tokens has shape [batch, n_tokens, d_model] + - modality_id is an integer index for the modality embedding + + Returns + ------- + torch.Tensor + Transformer output of shape [batch, total_tokens, d_model] + """ + B = token_list[0][0].shape[0] + device = token_list[0][0].device + + # Concatenate all modality tokens + all_tokens = [] + for tokens, modality_id in token_list: + # Add modality embedding + mod_emb = self.modality_embeddings( + torch.tensor(modality_id, device=device) + ) + tokens = tokens + mod_emb + all_tokens.append(tokens) + + x = torch.cat(all_tokens, dim=1) # [B, total_tokens, d_model] + + # Add positional embeddings + n_tokens = x.shape[1] + positions = torch.arange(n_tokens, device=device) + x = x + self.position_embeddings(positions) + + # Causal mask + mask = self._causal_mask(n_tokens, device) + + # Transformer forward pass + x = self.transformer(x, mask=mask) # [B, total_tokens, d_model] + + return x + + +if __name__ == "__main__": + d_model = 512 + B = 4 + + transformer = BaselineFusionTransformer( + d_model=d_model, + n_heads=8, + n_layers=6, + n_modalities=7, + max_tokens=1024, + verbose=True + ) + + # Dummy encoder outputs + ts_tokens = torch.randn(B, 100, d_model) # TimeSeriesEncoder + sp_tokens = torch.randn(B, 10, d_model) # SpatialProfileEncoder + vid_tokens = torch.randn(B, 192, d_model) # VideoEncoder (VIS) + ir_tokens = torch.randn(B, 192, d_model) # VideoEncoder (IR) + spec_tokens = torch.randn(B, 50, d_model) # SpectrogramEncoder + text_tokens = torch.randn(B, 20, d_model) # TextEncoder + + token_list = [ + (ts_tokens, 0), # modality 0: time series + (sp_tokens, 1), # modality 1: spatial profile + (vid_tokens, 2), # modality 2: visible camera + (ir_tokens, 3), # modality 3: IR camera + (spec_tokens, 4), # modality 4: spectrogram + (text_tokens, 5), # modality 5: text + ] + + out = transformer(token_list) + print(f"Input tokens: {sum(t.shape[1] for t, _ in token_list)}") # 564 + print(f"Output shape: {out.shape}") # [4, 564, 512] From 1fc8fc43351fdbf699c015b708ac3ccf8554aab4 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Tue, 17 Feb 2026 09:46:50 -0500 Subject: [PATCH 14/30] Foundation model (#56) * Nathan fm (#53) * chore: Update `pyproject.toml` to reorder authors, enhance README with environment setup instructions, and add validation notes in `validation.txt`. Refactor `dummy_model_2.py` for improved modality configuration and introduce `TextEncoder` enhancements in `text_baseline.py`. * Refactor demo scripts to utilize new `Prediction4FusionModel` and `DictMSELoss`. Update `run_demo_2.py` and `run_demo_3.py` for improved model initialization and data handling. Enhance `TokamakH5Dataset` to handle degenerate signals and improve data extraction logic. Remove unused `latent_space.py` and integrate new modality fusion models in `modality_fusion.py`. * Remove unused shot list configuration files and refactor trainer class to introduce MultimodalTrainer and UnimodalTrainer for improved training structure. * Refactor modality models and trainer classes for improved structure and functionality. Removed unused TimeSeriesEncoder and Decoder, introduced FastTimeSeriesEncoder and SpectrogramAutoEncoder. Updated UnimodalTrainer to support logging and checkpoint management. Enhanced TokamakH5Dataset for better data handling and added checkpoint loading functionality in spectrogram reconstruction script. * Add padding collate function and update training script for unimodal autoencoder - Introduced `collate_fn_pad` to handle variable-length tensors in batches. - Updated `train_unimodal_autoencoder.py` to use the new collate function. - Modified `train_unimodal.sh` to include additional signal modalities for training. - Added new autoencoder classes for fast time series and spatial profile modalities, ensuring output shape consistency with adaptive pooling. - Enhanced video autoencoder implementation for better reconstruction quality. * Remove spectrogram reconstruction script and refactor modality models - Deleted `spectrogram_reconstruction.py` as part of the restructuring. - Refactored modality models to introduce baseline versions for actuator, slow time series, fast time series, spatial profile, spectrogram, and video. - Updated model registry and signal-to-model mappings to reflect new baseline architecture. - Enhanced `TokamakH5Dataset` to support additional parameters for FFT and hop length. - Improved training script for unimodal autoencoders to utilize new baseline models and added support for variable-length tensors. * Update .gitignore to include pixi environments and add link to HSI-compression-benchmark in SpectrogramBaselineAutoEncoder docstring * Remove unused shot list files and delete deprecated scripts for training and data handling * Remove deprecated training scripts for CO2, ECE, MHR, and unimodal training * Dev peter (#48) * Removed the argument "batch_size" from the trainers. Changed default hyperparameters in the models. Added demo for profile reconstruction. Added script for dataset standardization (has to be run once before model training to store normalization coefficients). * Bugfix in the dataset class. When iterating over movie configurations, the wrong configuration was used to find the correct signal name. Also, removed warning for duplicated tensor conversion. * Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. * Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. * Minor changes in the example scripts. More preprocessing options for the dataset class. * Fixed a bug where the dataset class failed when using multiple workers and opening an H5 file prior to distributing the dataset across all workers. Significant updates in the Fast time series baseline and actuator reconstruction classes. * Lots of bugfixes in the dataset, trainer, and models. The basic encoders are now all working. Examples are in scripts. * Dev peter (#50) * Removed the argument "batch_size" from the trainers. Changed default hyperparameters in the models. Added demo for profile reconstruction. Added script for dataset standardization (has to be run once before model training to store normalization coefficients). * Bugfix in the dataset class. When iterating over movie configurations, the wrong configuration was used to find the correct signal name. Also, removed warning for duplicated tensor conversion. * Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. * Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. * Minor changes in the example scripts. More preprocessing options for the dataset class. * Fixed a bug where the dataset class failed when using multiple workers and opening an H5 file prior to distributing the dataset across all workers. Significant updates in the Fast time series baseline and actuator reconstruction classes. * Lots of bugfixes in the dataset, trainer, and models. The basic encoders are now all working. Examples are in scripts. * Extended checkpointing - the trainer stores now: - Model - Optimizer state - Scheduler state - Current loss - Current epoch For the sake of continual training. * Extended checkpointing - the trainer stores now: - Model - Optimizer state - Scheduler state - Current loss - Current epoch For the sake of continual training. * Adapted the other reconstruction scripts to match the new API. * Bugfix in the dataset class. When splitting inputs and targets, I forgot to remove unused modalities. This follows the standard getitem function now. * Prepared an option to preprocess movies. This has to be fully integrated!!! --------- Co-authored-by: Peter Steiner <61472983+renierts@users.noreply.github.com> * Dev peter (#55) * Removed the argument "batch_size" from the trainers. Changed default hyperparameters in the models. Added demo for profile reconstruction. Added script for dataset standardization (has to be run once before model training to store normalization coefficients). * Bugfix in the dataset class. When iterating over movie configurations, the wrong configuration was used to find the correct signal name. Also, removed warning for duplicated tensor conversion. * Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. * Added base script for video reconstruction. Copied from Aza's branch for debugging purposes. * Minor changes in the example scripts. More preprocessing options for the dataset class. * Fixed a bug where the dataset class failed when using multiple workers and opening an H5 file prior to distributing the dataset across all workers. Significant updates in the Fast time series baseline and actuator reconstruction classes. * Lots of bugfixes in the dataset, trainer, and models. The basic encoders are now all working. Examples are in scripts. * Extended checkpointing - the trainer stores now: - Model - Optimizer state - Scheduler state - Current loss - Current epoch For the sake of continual training. * Extended checkpointing - the trainer stores now: - Model - Optimizer state - Scheduler state - Current loss - Current epoch For the sake of continual training. * Adapted the other reconstruction scripts to match the new API. * Bugfix in the dataset class. When splitting inputs and targets, I forgot to remove unused modalities. This follows the standard getitem function now. * Prepared an option to preprocess movies. This has to be fully integrated!!! * Added a baseline fusion transformer for latent space prediction. Quick fix for the data standardization. Invalid values have to be ignored. Fix in the function to create H5 files. bolo data does not have to be flipped anymore as the data is now stored in the correct format. --------- Co-authored-by: Nathaniel Chen --- .gitignore | 3 + .../check_dataset_integrity.py | 38 + .../convert_to_training_data.py | 0 scripts/{ => data_preparation}/fetch_data.py | 0 .../data_preparation/make_processing_stats.py | 37 + scripts/{ => data_preparation}/mygadata.py | 0 scripts/notebooks/test_dataloader.ipynb | 890 ------------------ scripts/notebooks/test_model.ipynb | 387 -------- scripts/run_demo_2.py | 127 --- scripts/run_demo_3.py | 163 ---- .../slurm/train_bes.sh | 0 scripts/slurm/train_co2.sh | 25 + scripts/slurm/train_ece.sh | 28 + scripts/slurm/train_mhr.sh | 26 + scripts/slurm/train_unimodal.sh | 25 + scripts/train_unimodal.sh | 45 - .../{ => training}/actuator_reconstruction.py | 0 .../fast_time_series_reconstruction.py | 190 ++++ scripts/training/profile_reconstruction.py | 194 ++++ scripts/{ => training}/run_demo.py | 0 .../train_unimodal_autoencoder.py | 13 +- .../{ => training}/video_reconstruction.py | 0 .../data/config/config.yaml | 7 + .../data/config/modalities/modalities.yaml | 138 +++ .../data/config/shot_list/train_debug.yaml | 11 + .../data/config/shot_list/train_full.txt | 0 .../data/config/shot_list/train_medium.txt | 0 .../data/config/shot_list/train_small.yaml | 640 +++++++++++++ .../data/config}/shot_list/validation.txt | 0 .../data/data_loader.py | 40 +- .../data/dummy_data.py | 12 +- .../data/prepare_data.py | 162 ++++ .../models/modality/cer_model.py | 84 ++ .../models/modality/spectrogram_baseline.py | 61 +- .../models/modality/spectrogram_cae1d.py | 234 +++++ .../trainer/trainer.py | 13 +- src/tokamak_foundation_model/utils/drawing.py | 6 +- 37 files changed, 1965 insertions(+), 1634 deletions(-) create mode 100644 scripts/data_preparation/check_dataset_integrity.py rename config/shot_list/train_debug.txt => scripts/data_preparation/convert_to_training_data.py (100%) rename scripts/{ => data_preparation}/fetch_data.py (100%) create mode 100644 scripts/data_preparation/make_processing_stats.py rename scripts/{ => data_preparation}/mygadata.py (100%) delete mode 100644 scripts/notebooks/test_dataloader.ipynb delete mode 100644 scripts/notebooks/test_model.ipynb delete mode 100644 scripts/run_demo_2.py delete mode 100644 scripts/run_demo_3.py rename config/shot_list/train_full.txt => scripts/slurm/train_bes.sh (100%) create mode 100644 scripts/slurm/train_co2.sh create mode 100644 scripts/slurm/train_ece.sh create mode 100644 scripts/slurm/train_mhr.sh create mode 100644 scripts/slurm/train_unimodal.sh delete mode 100644 scripts/train_unimodal.sh rename scripts/{ => training}/actuator_reconstruction.py (100%) create mode 100644 scripts/training/fast_time_series_reconstruction.py create mode 100644 scripts/training/profile_reconstruction.py rename scripts/{ => training}/run_demo.py (100%) rename scripts/{ => training}/train_unimodal_autoencoder.py (91%) rename scripts/{ => training}/video_reconstruction.py (100%) create mode 100644 src/tokamak_foundation_model/data/config/config.yaml create mode 100644 src/tokamak_foundation_model/data/config/modalities/modalities.yaml create mode 100644 src/tokamak_foundation_model/data/config/shot_list/train_debug.yaml rename config/shot_list/train_medium.txt => src/tokamak_foundation_model/data/config/shot_list/train_full.txt (100%) rename config/shot_list/train_small.txt => src/tokamak_foundation_model/data/config/shot_list/train_medium.txt (100%) create mode 100644 src/tokamak_foundation_model/data/config/shot_list/train_small.yaml rename {config => src/tokamak_foundation_model/data/config}/shot_list/validation.txt (100%) create mode 100644 src/tokamak_foundation_model/data/prepare_data.py create mode 100644 src/tokamak_foundation_model/models/modality/cer_model.py create mode 100644 src/tokamak_foundation_model/models/modality/spectrogram_cae1d.py diff --git a/.gitignore b/.gitignore index e15106e..01458f5 100644 --- a/.gitignore +++ b/.gitignore @@ -214,3 +214,6 @@ __marimo__/ # Streamlit .streamlit/secrets.toml +# pixi environments +.pixi/* +!.pixi/config.toml diff --git a/scripts/data_preparation/check_dataset_integrity.py b/scripts/data_preparation/check_dataset_integrity.py new file mode 100644 index 0000000..60dc48d --- /dev/null +++ b/scripts/data_preparation/check_dataset_integrity.py @@ -0,0 +1,38 @@ +from pathlib import Path +from tqdm.auto import tqdm +from torch.utils.data import ConcatDataset +from tokamak_foundation_model.data.data_loader import ( + TokamakH5Dataset) + +# hdf5_files = sorted( +# Path( +# "/scratch/gpfs/EKOLEMEN/foundation_model" +# ).glob("*_processed.h5") +# ) + +hdf5_files = sorted( + Path( + "/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data" + ).glob("*_processed.h5") +) + +all_input_signals = [ + "mhr", "ece", "co2", "bes", # spectrograms + "gas", "ech", "pin", "tin", # actuators + "d_alpha", "mse", "ts_core_density", # diagnostics + "bolo", "irtv", "tangtv", # videos + # "text", # metadata +] + +datasets = [ + TokamakH5Dataset( + hdf5_path=str(f), + input_signals=all_input_signals, + target_signals=all_input_signals, + ) for f in hdf5_files[:1]] + +datasets = ConcatDataset(datasets) + +for i in tqdm(range(len(datasets))): + print(datasets[i]['inputs'].keys()) + break \ No newline at end of file diff --git a/config/shot_list/train_debug.txt b/scripts/data_preparation/convert_to_training_data.py similarity index 100% rename from config/shot_list/train_debug.txt rename to scripts/data_preparation/convert_to_training_data.py diff --git a/scripts/fetch_data.py b/scripts/data_preparation/fetch_data.py similarity index 100% rename from scripts/fetch_data.py rename to scripts/data_preparation/fetch_data.py diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py new file mode 100644 index 0000000..53bc61f --- /dev/null +++ b/scripts/data_preparation/make_processing_stats.py @@ -0,0 +1,37 @@ +from pathlib import Path +from tokamak_foundation_model.data.data_loader import ( + TokamakH5Dataset, compute_preprocessing_stats) + +def main(): + # hdf5_files = sorted( + # Path( + # "/scratch/gpfs/EKOLEMEN/foundation_model" + # ).glob("*_processed.h5") + # ) + + hdf5_files = sorted( + Path( + "/scratch/gpfs/EKOLEMEN/foundation_model" + ).glob("*_processed.h5") + ) + + all_input_signals = [ + "mhr", "ece", "co2", "bes", # spectrograms + "gas", "ech", "pin", "tin", # actuators + "d_alpha", "mse", "ts_core_density", # diagnostics + "bolo", "irtv", "tangtv", # videos + # "text", # metadata + ] + + datasets = [ + TokamakH5Dataset( + hdf5_path=str(f), + input_signals=all_input_signals, + target_signals=all_input_signals, + ) for f in hdf5_files] + + stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') + +if __name__ == "__main__": + # python scripts/data_preparation/make_processing_stats.py + main() \ No newline at end of file diff --git a/scripts/mygadata.py b/scripts/data_preparation/mygadata.py similarity index 100% rename from scripts/mygadata.py rename to scripts/data_preparation/mygadata.py diff --git a/scripts/notebooks/test_dataloader.ipynb b/scripts/notebooks/test_dataloader.ipynb deleted file mode 100644 index 5d77a7e..0000000 --- a/scripts/notebooks/test_dataloader.ipynb +++ /dev/null @@ -1,890 +0,0 @@ -{ - "cells": [ - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:54.827879Z", - "start_time": "2025-08-03T13:32:53.423730Z" - } - }, - "cell_type": "code", - "source": [ - "import numpy as np\n", - "from joblib import load\n", - "\n", - "import torch\n", - "from torch.utils.data import Dataset, DataLoader, get_worker_info\n", - "\n", - "from pathlib import Path\n", - "\n", - "import cProfile" - ], - "id": "7fb8dbfbb30ebc5d", - "outputs": [], - "execution_count": 1 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:54.974920Z", - "start_time": "2025-08-03T13:32:54.961905Z" - } - }, - "cell_type": "code", - "source": [ - "import matplotlib.pyplot as plt\n", - "import seaborn as sns\n", - "\n", - "\n", - "sns.set_theme()\n", - "%matplotlib inline" - ], - "id": "5328e1c94331c459", - "outputs": [], - "execution_count": 2 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:54.989560Z", - "start_time": "2025-08-03T13:32:54.981960Z" - } - }, - "cell_type": "code", - "source": [ - "class JoblibDataset(Dataset): # TODO: Dictionary outputs!!!\n", - "\n", - " def __init__(self, file_paths, subseq_len, keys=['mhr', 'ece', 'co2']):\n", - " super().__init__()\n", - " self.file_paths = file_paths\n", - " self.subseq_len = subseq_len\n", - " self.keys = keys\n", - " self.data_shapes = None\n", - "\n", - " # Don't open the files here; just build an index (subseq_info) of what subsequences exist.\n", - " self._opened_files = None # This remains None until worker_init_fn calls .worker_init()\n", - "\n", - " self.subseq_info = []\n", - " # Build an index of all non-overlapping chunks across all files by reading shapes only.\n", - " for f_idx, fp in enumerate(self.file_paths):\n", - " # Load enough info to get n_samples\n", - " data_dict = load(fp, mmap_mode='r')\n", - " if self.data_shapes is None:\n", - " self.data_shapes = {key: data_dict[key].shape[:-1] + (subseq_len, ) for key in keys}\n", - " all_n_samples = [data_dict[key].shape[-1] for key in keys]\n", - " n_samples = np.min(all_n_samples)\n", - " # Drop the reference to close the file handle\n", - " del data_dict\n", - "\n", - " # If we can fit at least one subsequence\n", - " if self.subseq_len == -1:\n", - " self.subseq_info.append((f_idx, 0))\n", - " continue\n", - " if n_samples >= self.subseq_len:\n", - " n_chunks = n_samples // self.subseq_len\n", - " for chunk_idx in range(n_chunks):\n", - " start_samp = chunk_idx * self.subseq_len\n", - " self.subseq_info.append((f_idx, start_samp))\n", - "\n", - " # The total number of subsequences across all files\n", - " self.total_subseqs = len(self.subseq_info)\n", - "\n", - " def __len__(self):\n", - " return self.total_subseqs\n", - "\n", - " def worker_init(self):\n", - " self._opened_files = []\n", - " for fp in self.file_paths:\n", - " data_dict = load(fp, mmap_mode='r')\n", - " filtered_dict = {key: data_dict[key] for key in self.keys}\n", - " self._opened_files.append(filtered_dict)\n", - "\n", - " def _load_single_sample(self, idx):\n", - " \"\"\"\n", - " Load a single sample from the opened files.\n", - " This is used in __getitem__ and __getitems__.\n", - " \"\"\"\n", - " if self._opened_files is None:\n", - " self.worker_init()\n", - "\n", - " file_idx, start_samp = self.subseq_info[idx]\n", - "\n", - " data_dict = self._opened_files[file_idx]\n", - "\n", - " # Slicing out a non-overlapping subsequence\n", - " end_samp = start_samp + self.subseq_len\n", - " \"\"\"\n", - " inputs = dict()\n", - " for key in self.keys:\n", - " inputs[key] = torch.from_numpy(data_dict[key][..., start_samp:end_samp])\n", - " \"\"\"\n", - " inputs = [torch.from_numpy(data_dict[key][..., start_samp:end_samp]) for key in self.keys]\n", - "\n", - " return inputs\n", - "\n", - " def __getitem__(self, idx):\n", - " data = self._load_single_sample(idx)\n", - " return {key: val for key, val in zip(self.keys, data)}\n", - "\n", - " def __getitems__(self, indices):\n", - " \"\"\"\n", - " Optimized batch loading using __getitems__\n", - " This method is called automatically when using BatchSampler\n", - " \"\"\"\n", - " batch_size = len(indices)\n", - " \n", - " # Load all samples using the existing __getitem__ logic\n", - " samples = [self._load_single_sample(idx) for idx in indices]\n", - " \"\"\"\n", - " # Find C dimensions for this batch\n", - " C = [sample.shape[0] for sample in samples[0]]\n", - " \n", - " # Get F, T from first sample (assuming they're consistent)\n", - " F, T = samples[0][0].shape[1], samples[0][0].shape[2]\n", - " \"\"\"\n", - " batch_tensors = [\n", - " torch.empty((batch_size, *shape), dtype=torch.float32) for shape in self.data_shapes.values()\n", - " ]\n", - " \n", - " for i, sample in enumerate(samples):\n", - " for j, s in enumerate(sample):\n", - " batch_tensors[j][i] = s\n", - " return {key: val for key, val in zip(self.keys, batch_tensors)}" - ], - "id": "5a1c67d44ac865d1", - "outputs": [], - "execution_count": 3 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:54.998833Z", - "start_time": "2025-08-03T13:32:54.995611Z" - } - }, - "cell_type": "code", - "source": [ - "def my_worker_init_fn(worker_id):\n", - " # This is called in each worker process once\n", - " worker_info = get_worker_info()\n", - " dataset = worker_info.dataset\n", - " dataset.worker_init() # Actually open the joblib files in this worker" - ], - "id": "227f8c4de55a75de", - "outputs": [], - "execution_count": 4 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:55.009526Z", - "start_time": "2025-08-03T13:32:55.006593Z" - } - }, - "cell_type": "code", - "source": [ - "def collate_fn(batch):\n", - " \"\"\"\n", - " Custom collate function for __getitems__ approach\n", - " \"\"\"\n", - " # __getitems__ returns a list with one element containing batched tensors\n", - " return batch" - ], - "id": "60a63a7e256ad186", - "outputs": [], - "execution_count": 5 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:55.030568Z", - "start_time": "2025-08-03T13:32:55.018060Z" - } - }, - "cell_type": "code", - "source": [ - "%%time\n", - "# training_files_dir = Path('/scratch/gpfs/EKOLEMEN/hackathon/foundation25/spectrogram/')\n", - "training_files_dir = Path('./')\n", - "training_set = JoblibDataset(list(training_files_dir.glob(\"*.joblib\"))[:500],\n", - " subseq_len=256)\n", - "\n", - "train_loader = DataLoader(\n", - " training_set,\n", - " batch_size=16,\n", - " # pin_memory=True,\n", - " shuffle=True,\n", - " num_workers=0,\n", - " # persistent_workers=True,\n", - " collate_fn=collate_fn,\n", - " worker_init_fn=my_worker_init_fn\n", - ")\n", - "\n", - "# train_loader = DataLoader(training_set, batch_size=16, pin_memory=True, shuffle=False,\n", - "# num_workers=0, worker_init_fn=my_worker_init_fn)" - ], - "id": "696483fef26771fa", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: total: 0 ns\n", - "Wall time: 0 ns\n" - ] - } - ], - "execution_count": 6 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:55.051159Z", - "start_time": "2025-08-03T13:32:55.047230Z" - } - }, - "cell_type": "code", - "source": [ - "def iterate_train_loader():\n", - " for batch_data in train_loader:\n", - " for data in batch_data:\n", - " pass" - ], - "id": "f715836781fc2501", - "outputs": [], - "execution_count": 7 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:56.475805Z", - "start_time": "2025-08-03T13:32:55.074038Z" - } - }, - "cell_type": "code", - "source": [ - "profiler = cProfile.Profile()\n", - "profiler.enable()\n", - "iterate_train_loader() # Call the function to profile\n", - "profiler.disable()\n", - "profiler.print_stats(sort='cumulative') # Sort by cumulative time" - ], - "id": "a16ff0ab8f6dd5a8", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_114504\\3480888288.py:67: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at C:\\actions-runner\\_work\\pytorch\\pytorch\\pytorch\\torch\\csrc\\utils\\tensor_numpy.cpp:209.)\n", - " inputs = [torch.from_numpy(data_dict[key][..., start_samp:end_samp]) for key in self.keys]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 15114 function calls (14979 primitive calls) in 1.379 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 8 0.000 0.000 1.178 0.147 dataloader.py:728(__next__)\n", - " 8 0.000 0.000 1.177 0.147 dataloader.py:787(_next_data)\n", - " 7 0.000 0.000 1.177 0.168 fetch.py:47(fetch)\n", - " 7 1.175 0.168 1.177 0.168 3480888288.py:75(__getitems__)\n", - " 6 0.000 0.000 0.200 0.033 base_events.py:1908(_run_once)\n", - " 6 0.000 0.000 0.199 0.033 selectors.py:319(select)\n", - " 6 0.000 0.000 0.199 0.033 selectors.py:313(_select)\n", - " 97 0.000 0.000 0.013 0.000 3480888288.py:48(_load_single_sample)\n", - " 1 0.000 0.000 0.011 0.011 3480888288.py:41(worker_init)\n", - " 4 0.000 0.000 0.011 0.003 numpy_pickle.py:674(load)\n", - " 4 0.000 0.000 0.010 0.003 numpy_pickle.py:613(_unpickle)\n", - " 4 0.001 0.000 0.010 0.003 pickle.py:1179(load)\n", - " 40 0.000 0.000 0.008 0.000 numpy_pickle.py:438(load_build)\n", - " 32 0.000 0.000 0.008 0.000 numpy_pickle.py:259(read)\n", - " 32 0.000 0.000 0.006 0.000 numpy_pickle.py:215(read_mmap)\n", - " 32 0.000 0.000 0.006 0.000 backports.py:113(make_memmap)\n", - " 32 0.001 0.000 0.005 0.000 memmap.py:216(__new__)\n", - " 36 0.004 0.000 0.004 0.000 {built-in method _io.open}\n", - " 1836 0.001 0.000 0.001 0.000 pickle.py:281(read)\n", - " 11 0.000 0.000 0.001 0.000 events.py:86(_run)\n", - " 23 0.001 0.000 0.001 0.000 {built-in method torch.empty}\n", - " 11 0.000 0.000 0.001 0.000 {method 'run' of '_contextvars.Context' objects}\n", - " 8 0.000 0.000 0.001 0.000 profiler.py:776(__exit__)\n", - " 291 0.000 0.000 0.001 0.000 memmap.py:359(__getitem__)\n", - " 9 0.000 0.000 0.001 0.000 ioloop.py:750(_run_callback)\n", - " 9 0.000 0.000 0.001 0.000 iostream.py:616(_flush)\n", - " 291 0.000 0.000 0.001 0.000 {built-in method torch.from_numpy}\n", - " 1 0.000 0.000 0.001 0.001 dataloader.py:480(__iter__)\n", - " 288 0.000 0.000 0.001 0.000 pickle.py:1609(load_binget)\n", - " 21 0.000 0.000 0.001 0.000 {built-in method builtins.next}\n", - " 8 0.000 0.000 0.001 0.000 _ops.py:981(__call__)\n", - " 323 0.000 0.000 0.000 0.000 memmap.py:312(__array_finalize__)\n", - "1658/1654 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", - " 8 0.000 0.000 0.000 0.000 dataloader.py:722(_next_index)\n", - " 2 0.000 0.000 0.000 0.000 asyncio.py:206(_handle_events)\n", - " 2 0.000 0.000 0.000 0.000 zmqstream.py:573(_handle_events)\n", - " 8 0.000 0.000 0.000 0.000 sampler.py:326(__iter__)\n", - " 1 0.000 0.000 0.000 0.000 session.py:754(send)\n", - " 821 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects}\n", - " 98 0.000 0.000 0.000 0.000 sampler.py:167(__iter__)\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:1035(_must_dispatch_in_python)\n", - " 8 0.000 0.000 0.000 0.000 profiler.py:770(__enter__)\n", - " 8 0.000 0.000 0.000 0.000 _pytree.py:1410(tree_any)\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:1147(__call__)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:419(_get_iterator)\n", - " 10 0.000 0.000 0.000 0.000 {built-in method builtins.any}\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:766(__init__)\n", - " 64 0.000 0.000 0.000 0.000 backports.py:72(__init__)\n", - " 9 0.000 0.000 0.000 0.000 socket.py:623(send)\n", - " 40 0.000 0.000 0.000 0.000 pickle.py:1704(load_build)\n", - " 2 0.000 0.000 0.000 0.000 iostream.py:259(schedule)\n", - " 144 0.000 0.000 0.000 0.000 {method 'read' of '_io.BufferedReader' objects}\n", - " 64 0.000 0.000 0.000 0.000 backports.py:76(parse)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:634(__init__)\n", - " 36 0.000 0.000 0.000 0.000 :236(split)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method torch._ops.profiler._record_function_enter_new}\n", - " 2185 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", - " 8 0.000 0.000 0.000 0.000 {built-in method torch._ops.profiler.}\n", - " 32 0.000 0.000 0.000 0.000 :268(basename)\n", - " 56/16 0.000 0.000 0.000 0.000 _pytree.py:1071(tree_iter)\n", - " 1788 0.000 0.000 0.000 0.000 {method 'read' of '_io.BytesIO' objects}\n", - " 2 0.000 0.000 0.000 0.000 zmqstream.py:614(_handle_recv)\n", - " 36 0.000 0.000 0.000 0.000 {method '__exit__' of '_io._IOBase' objects}\n", - " 92 0.000 0.000 0.000 0.000 pickle.py:1417(load_short_binunicode)\n", - " 65 0.000 0.000 0.000 0.000 {built-in method __new__ of type object at 0x00007FFFE1C098B0}\n", - " 8 0.000 0.000 0.000 0.000 profiler.py:759(__init__)\n", - " 1 0.000 0.000 0.000 0.000 warnings.py:97(_showwarnmsg)\n", - " 1 0.000 0.000 0.000 0.000 warnings.py:20(_showwarnmsg_impl)\n", - " 15/14 0.000 0.000 0.000 0.000 typing.py:378(inner)\n", - " 6 0.000 0.000 0.000 0.000 contextlib.py:132(__enter__)\n", - " 1 0.000 0.000 0.000 0.000 session.py:690(serialize)\n", - " 35 0.000 0.000 0.000 0.000 :117(__instancecheck__)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method torch.randperm}\n", - " 2 0.000 0.000 0.000 0.000 {method 'random_' of 'torch._C.TensorBase' objects}\n", - " 1 0.000 0.000 0.000 0.000 interactiveshell.py:3043(write)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:271(send_multipart)\n", - " 32 0.000 0.000 0.000 0.000 backports.py:35(__lt__)\n", - " 2 0.000 0.000 0.000 0.000 zmqstream.py:546(_run_callback)\n", - " 35 0.000 0.000 0.000 0.000 {built-in method _abc._abc_instancecheck}\n", - " 8 0.000 0.000 0.000 0.000 numpy_pickle_utils.py:130(_validate_fileobject_and_memmap)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:655(write)\n", - " 2 0.000 0.000 0.000 0.000 iostream.py:157(_handle_event)\n", - " 45/5 0.000 0.000 0.000 0.000 :121(__subclasscheck__)\n", - " 32 0.000 0.000 0.000 0.000 backports.py:96(_cmp)\n", - " 4 0.000 0.000 0.000 0.000 numpy_pickle_utils.py:88(_detect_compressor)\n", - " 2/1 0.000 0.000 0.000 0.000 typing.py:494(__getitem__)\n", - " 45/5 0.000 0.000 0.000 0.000 {built-in method _abc._abc_subclasscheck}\n", - " 32 0.000 0.000 0.000 0.000 :596(abspath)\n", - " 1 0.000 0.000 0.000 0.000 typing.py:726(Optional)\n", - " 2 0.000 0.000 0.000 0.000 {method '__exit__' of 'sqlite3.Connection' objects}\n", - " 64 0.000 0.000 0.000 0.000 {method 'seek' of '_io.BufferedReader' objects}\n", - " 10 0.000 0.000 0.000 0.000 iostream.py:710(_flush_buffers)\n", - " 36 0.000 0.000 0.000 0.000 pickle.py:1686(load_setitems)\n", - " 1 0.000 0.000 0.000 0.000 _ops.py:1089(__getattr__)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:577(_schedule_flush)\n", - " 3 0.000 0.000 0.000 0.000 traitlets.py:708(__set__)\n", - " 32 0.000 0.000 0.000 0.000 pickle.py:1228(load_frame)\n", - " 4 0.000 0.000 0.000 0.000 attrsettr.py:43(__getattr__)\n", - " 100 0.000 0.000 0.000 0.000 pickle.py:1276(load_binint1)\n", - " 3 0.000 0.000 0.000 0.000 {built-in method builtins.compile}\n", - " 228 0.000 0.000 0.000 0.000 pickle.py:1648(load_memoize)\n", - " 64 0.000 0.000 0.000 0.000 {method 'split' of 're.Pattern' objects}\n", - " 4 0.000 0.000 0.000 0.000 session.py:92(json_packer)\n", - " 3 0.000 0.000 0.000 0.000 traitlets.py:689(set)\n", - " 452 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 1 0.000 0.000 0.000 0.000 2121102005.py:1()\n", - " 32 0.000 0.000 0.000 0.000 _pytree.py:836(_is_leaf)\n", - " 2 0.000 0.000 0.000 0.000 codeop.py:121(__call__)\n", - " 56 0.000 0.000 0.000 0.000 _pytree.py:829(_get_node_type)\n", - " 12 0.000 0.000 0.000 0.000 pickle.py:1525(load_stack_global)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:276()\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:278(_really_send)\n", - " 4 0.000 0.000 0.000 0.000 __init__.py:183(dumps)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 32 0.000 0.000 0.000 0.000 {method 'format' of 'str' objects}\n", - " 36 0.000 0.000 0.000 0.000 :179(splitroot)\n", - " 44 0.000 0.000 0.000 0.000 pickle.py:1280(load_binint2)\n", - " 192 0.000 0.000 0.000 0.000 {built-in method sys.intern}\n", - " 1 0.000 0.000 0.000 0.000 socket.py:700(send_multipart)\n", - " 32 0.000 0.000 0.000 0.000 pickle.py:1503(load_newobj)\n", - " 64 0.000 0.000 0.000 0.000 {method 'tell' of '_io.BufferedReader' objects}\n", - " 1 0.000 0.000 0.000 0.000 typing.py:673(Union)\n", - " 4 0.000 0.000 0.000 0.000 attrsettr.py:66(_get_attr_opt)\n", - " 3 0.000 0.000 0.000 0.000 traitlets.py:718(_validate)\n", - " 56 0.000 0.000 0.000 0.000 _pytree.py:818(_is_namedtuple_instance)\n", - " 4 0.000 0.000 0.000 0.000 encoder.py:183(encode)\n", - " 12 0.000 0.000 0.000 0.000 pickle.py:1564(find_class)\n", - " 1 0.000 0.000 0.000 0.000 session.py:649(msg)\n", - " 32 0.000 0.000 0.000 0.000 :564(normpath)\n", - " 4 0.000 0.000 0.000 0.000 numpy_pickle.py:420(__init__)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3631(set)\n", - " 4 0.000 0.000 0.000 0.000 {method 'peek' of '_io.BufferedReader' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'tolist' of 'torch._C.TensorBase' objects}\n", - " 3 0.000 0.000 0.000 0.000 typing.py:175(_type_check)\n", - " 20 0.000 0.000 0.000 0.000 traitlets.py:676(__get__)\n", - " 4 0.000 0.000 0.000 0.000 encoder.py:205(iterencode)\n", - " 1 0.000 0.000 0.000 0.000 session.py:675(sign)\n", - " 2 0.000 0.000 0.000 0.000 zmqstream.py:653(_rebuild_io_state)\n", - " 9 0.000 0.000 0.000 0.000 {built-in method _heapq.heappop}\n", - " 2 0.000 0.000 0.000 0.000 socket.py:771(recv_multipart)\n", - " 9 0.000 0.000 0.000 0.000 iostream.py:718(_rotate_buffers)\n", - " 92 0.000 0.000 0.000 0.000 {built-in method _struct.unpack}\n", - " 32 0.000 0.000 0.000 0.000 {built-in method nt._getfullpathname}\n", - " 2 0.000 0.000 0.000 0.000 {method 'item' of 'torch._C.TensorBase' objects}\n", - " 1 0.000 0.000 0.000 0.000 typing.py:1223(__init__)\n", - " 101 0.000 0.000 0.000 0.000 {built-in method builtins.getattr}\n", - " 1 0.000 0.000 0.000 0.000 session.py:645(msg_header)\n", - " 4 0.000 0.000 0.000 0.000 :275(dirname)\n", - " 216 0.000 0.000 0.000 0.000 {method 'pop' of 'list' objects}\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:587(_schedule_in_thread)\n", - " 4 0.000 0.000 0.000 0.000 numpy_pickle_utils.py:44(_get_prefixes_max_len)\n", - " 1 0.000 0.000 0.000 0.000 ioloop.py:604(call_later)\n", - " 3 0.000 0.000 0.000 0.000 typing.py:166(_type_convert)\n", - " 8 0.000 0.000 0.000 0.000 _pytree.py:603(_dict_flatten)\n", - " 44 0.000 0.000 0.000 0.000 pickle.py:1210(pop_mark)\n", - " 44 0.000 0.000 0.000 0.000 pickle.py:1728(load_mark)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method torch._C._get_operation_overload}\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:1037()\n", - " 1 0.000 0.000 0.000 0.000 typing.py:868(__init__)\n", - " 6 0.000 0.000 0.000 0.000 contextlib.py:299(helper)\n", - " 1 0.000 0.000 0.000 0.000 asyncio.py:216(call_at)\n", - " 40 0.000 0.000 0.000 0.000 pickle.py:1439(load_tuple3)\n", - " 16 0.000 0.000 0.000 0.000 pickle.py:1272(load_binint)\n", - " 32 0.000 0.000 0.000 0.000 pickle.py:307(load_frame)\n", - " 2 0.000 0.000 0.000 0.000 zmqstream.py:676(_update_handler)\n", - " 1 0.000 0.000 0.000 0.000 _ops.py:1044(_has_script_object_arg)\n", - " 13 0.000 0.000 0.000 0.000 base_events.py:732(time)\n", - " 291 0.000 0.000 0.000 0.000 multiarray.py:1418(may_share_memory)\n", - " 1 0.000 0.000 0.000 0.000 _ops.py:688(__init__)\n", - " 12 0.000 0.000 0.000 0.000 pickle.py:316(_getattribute)\n", - " 8 0.000 0.000 0.000 0.000 pickle.py:1578(load_reduce)\n", - " 20 0.000 0.000 0.000 0.000 enum.py:713(__call__)\n", - " 1 0.000 0.000 0.000 0.000 warnings.py:118(_formatwarnmsg)\n", - " 1 0.000 0.000 0.000 0.000 hmac.py:122(copy)\n", - " 1 0.000 0.000 0.000 0.000 queues.py:225(get)\n", - " 7 0.000 0.000 0.000 0.000 contextlib.py:141(__exit__)\n", - " 20 0.000 0.000 0.000 0.000 traitlets.py:629(get)\n", - " 1 0.000 0.000 0.000 0.000 base_events.py:741(call_later)\n", - " 21 0.000 0.000 0.000 0.000 events.py:127(__lt__)\n", - " 36 0.000 0.000 0.000 0.000 :35(_get_bothseps)\n", - " 2 0.000 0.000 0.000 0.000 threading.py:1220(is_alive)\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:2635(validate)\n", - " 1 0.000 0.000 0.000 0.000 warnings.py:35(_formatwarnmsg_impl)\n", - " 136 0.000 0.000 0.000 0.000 {built-in method nt.fspath}\n", - " 6 0.000 0.000 0.000 0.000 enum.py:1541(__and__)\n", - " 1 0.000 0.000 0.000 0.000 jsonutil.py:107(json_default)\n", - " 32 0.000 0.000 0.000 0.000 numpy_pickle.py:114(safe_get_numpy_array_alignment_bytes)\n", - " 1 0.000 0.000 0.000 0.000 session.py:272(msg_header)\n", - " 1 0.000 0.000 0.000 0.000 base_events.py:765(call_at)\n", - " 1 0.000 0.000 0.000 0.000 {method 'copy' of '_hashlib.HMAC' objects}\n", - " 3 0.000 0.000 0.000 0.000 traitlets.py:727(_cross_validate)\n", - " 1 0.000 0.000 0.000 0.000 typing_extensions.py:3177(_collect_parameters)\n", - " 18 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", - " 32 0.000 0.000 0.000 0.000 {built-in method nt._path_normpath}\n", - " 8 0.000 0.000 0.000 0.000 enum.py:1531(__or__)\n", - " 46 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 typing.py:1175(__instancecheck__)\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:1512(_notify_trait)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3474(validate)\n", - " 6 0.000 0.000 0.000 0.000 contextlib.py:104(__init__)\n", - " 12 0.000 0.000 0.000 0.000 {built-in method builtins.__import__}\n", - " 32 0.000 0.000 0.000 0.000 {built-in method from_bytes}\n", - " 36 0.000 0.000 0.000 0.000 pickle.py:1447(load_empty_dictionary)\n", - " 40 0.000 0.000 0.000 0.000 pickle.py:1257(load_true)\n", - " 6 0.000 0.000 0.000 0.000 typing.py:1169(__setattr__)\n", - " 37 0.000 0.000 0.000 0.000 {built-in method nt.getpid}\n", - " 8 0.000 0.000 0.000 0.000 pickle.py:1422(load_tuple)\n", - " 1 0.000 0.000 0.000 0.000 typing.py:1130(__init__)\n", - " 37 0.000 0.000 0.000 0.000 {method 'replace' of 'str' objects}\n", - " 32 0.000 0.000 0.000 0.000 pickle.py:1427(load_empty_tuple)\n", - " 6 0.000 0.000 0.000 0.000 selector_events.py:750(_process_events)\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:1523(notify_change)\n", - " 2 0.000 0.000 0.000 0.000 _ops.py:1045()\n", - " 2 0.000 0.000 0.000 0.000 threading.py:1153(_wait_for_tstate_lock)\n", - " 8 0.000 0.000 0.000 0.000 {method '__setstate__' of 'numpy.dtype' objects}\n", - " 2 0.000 0.000 0.000 0.000 typing.py:1446(__subclasscheck__)\n", - " 4 0.000 0.000 0.000 0.000 pickle.py:1734(load_stop)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:98(_get_distributed_settings)\n", - " 1 0.000 0.000 0.000 0.000 session.py:600(msg_id)\n", - " 1 0.000 0.000 0.000 0.000 queues.py:256(get_nowait)\n", - " 16 0.000 0.000 0.000 0.000 _pytree.py:575(_tuple_flatten)\n", - " 36 0.000 0.000 0.000 0.000 {method 'rstrip' of 'str' objects}\n", - " 20 0.000 0.000 0.000 0.000 enum.py:1116(__new__)\n", - " 32 0.000 0.000 0.000 0.000 {method 'fileno' of '_io.BufferedReader' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'isoformat' of 'datetime.datetime' objects}\n", - " 13 0.000 0.000 0.000 0.000 {built-in method time.monotonic}\n", - " 1 0.000 0.000 0.000 0.000 session.py:198(utcnow)\n", - " 2 0.000 0.000 0.000 0.000 zmqstream.py:532(sending)\n", - " 19 0.000 0.000 0.000 0.000 {method 'values' of 'dict' objects}\n", - " 3 0.000 0.000 0.000 0.000 typing.py:709()\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:1527(_notify_observers)\n", - " 1 0.000 0.000 0.000 0.000 events.py:111(__init__)\n", - " 1 0.000 0.000 0.000 0.000 hmac.py:161(hexdigest)\n", - " 28 0.000 0.000 0.000 0.000 {method 'startswith' of 'bytes' objects}\n", - " 4 0.000 0.000 0.000 0.000 :1390(_handle_fromlist)\n", - " 15 0.000 0.000 0.000 0.000 {method 'append' of 'collections.deque' objects}\n", - " 1 0.000 0.000 0.000 0.000 typing.py:331(_remove_dups_flatten)\n", - " 55 0.000 0.000 0.000 0.000 typing.py:2132(cast)\n", - " 24 0.000 0.000 0.000 0.000 pickle.py:1249(load_none)\n", - " 4 0.000 0.000 0.000 0.000 hmac.py:117(update)\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:2558(_validate_bounds)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method torch._C._get_schema}\n", - " 2 0.000 0.000 0.000 0.000 queue.py:97(empty)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3624(validate_elements)\n", - " 17 0.000 0.000 0.000 0.000 typing.py:918(__eq__)\n", - " 6 0.000 0.000 0.000 0.000 typing.py:1117(_is_dunder)\n", - " 32 0.000 0.000 0.000 0.000 util.py:48(debug)\n", - " 1 0.000 0.000 0.000 0.000 typing_extensions.py:3092(_has_generic_or_protocol_as_origin)\n", - " 1 0.000 0.000 0.000 0.000 {method 'manual_seed' of 'torch._C.Generator' objects}\n", - " 13 0.000 0.000 0.000 0.000 {method 'popleft' of 'collections.deque' objects}\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:75(create_fetcher)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass}\n", - " 1 0.000 0.000 0.000 0.000 events.py:36(__init__)\n", - " 1 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HMAC' objects}\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.min}\n", - " 13 0.000 0.000 0.000 0.000 {method 'split' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method now}\n", - " 1 0.000 0.000 0.000 0.000 session.py:281(extract_header)\n", - " 4 0.000 0.000 0.000 0.000 {method 'update' of '_hashlib.HMAC' objects}\n", - " 1 0.000 0.000 0.000 0.000 :1087(__subclasshook__)\n", - " 4 0.000 0.000 0.000 0.000 numpy_pickle_utils.py:38(_is_raw_file)\n", - " 1 0.000 0.000 0.000 0.000 {method 'getvalue' of '_io.StringIO' objects}\n", - " 1 0.000 0.000 0.000 0.000 distributed_c10d.py:1269(is_initialized)\n", - " 4 0.000 0.000 0.000 0.000 typing.py:1239(__hash__)\n", - " 1 0.000 0.000 0.000 0.000 history.py:1016(_writeout_output_cache)\n", - " 1 0.000 0.000 0.000 0.000 linecache.py:26(getline)\n", - " 5 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n", - " 4 0.000 0.000 0.000 0.000 pickle.py:1221(load_proto)\n", - " 10 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 35 0.000 0.000 0.000 0.000 :288(__subclasshook__)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method _thread.allocate_lock}\n", - " 1 0.000 0.000 0.000 0.000 __init__.py:14(is_available)\n", - " 10 0.000 0.000 0.000 0.000 {method '__exit__' of '_thread.RLock' objects}\n", - " 4 0.000 0.000 0.000 0.000 compilerop.py:180(extra_flags)\n", - " 1 0.000 0.000 0.000 0.000 typing_extensions.py:3114(_is_unpacked_typevartuple)\n", - " 2 0.000 0.000 0.000 0.000 sampler.py:160(num_samples)\n", - " 2 0.000 0.000 0.000 0.000 ioloop.py:549(time)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:725(_hooks)\n", - " 12 0.000 0.000 0.000 0.000 {built-in method sys.audit}\n", - " 1 0.000 0.000 0.000 0.000 :104(_check_methods)\n", - " 8 0.000 0.000 0.000 0.000 pickle.py:1253(load_false)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:216(_check_mp_mode)\n", - " 8 0.000 0.000 0.000 0.000 {method '__exit__' of 'torch._C.DisableTorchFunctionSubclass' objects}\n", - " 4 0.000 0.000 0.000 0.000 pickle.py:1131(__init__)\n", - " 7 0.000 0.000 0.000 0.000 {method 'startswith' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:2304(validate)\n", - " 2 0.000 0.000 0.000 0.000 iostream.py:138(_event_pipe)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:505(parent_header)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.iter}\n", - " 1 0.000 0.000 0.000 0.000 warnings.py:419(__init__)\n", - " 1 0.000 0.000 0.000 0.000 typing.py:317(_deduplicate)\n", - " 2 0.000 0.000 0.000 0.000 base_events.py:1893(_add_callback)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method builtins.locals}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method _heapq.heappush}\n", - " 2 0.000 0.000 0.000 0.000 typing.py:927(__hash__)\n", - " 8 0.000 0.000 0.000 0.000 {method 'keys' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 _ops.py:55(__init__)\n", - " 7 0.000 0.000 0.000 0.000 3285181121.py:1(collate_fn)\n", - " 4 0.000 0.000 0.000 0.000 encoder.py:105(__init__)\n", - " 2 0.000 0.000 0.000 0.000 queue.py:209(_qsize)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:213(_is_master_process)\n", - " 1 0.000 0.000 0.000 0.000 linecache.py:36(getlines)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method builtins.setattr}\n", - " 8 0.000 0.000 0.000 0.000 _jit_internal.py:101(is_scripting)\n", - " 1 0.000 0.000 0.000 0.000 threading.py:314(_is_owned)\n", - " 4 0.000 0.000 0.000 0.000 pickle.py:98(__init__)\n", - " 6 0.000 0.000 0.000 0.000 {built-in method builtins.hash}\n", - " 1 0.000 0.000 0.000 0.000 fetch.py:9(__init__)\n", - " 2 0.000 0.000 0.000 0.000 interactiveshell.py:3615(compare)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:550(_is_master_process)\n", - " 4 0.000 0.000 0.000 0.000 {method '__exit__' of '_thread.lock' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'close' of '_io.StringIO' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'upper' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 typing.py:2288(get_origin)\n", - " 1 0.000 0.000 0.000 0.000 threading.py:299(__enter__)\n", - " 1 0.000 0.000 0.000 0.000 threading.py:308(_release_save)\n", - " 1 0.000 0.000 0.000 0.000 queues.py:173(qsize)\n", - " 1 0.000 0.000 0.000 0.000 distributed_c10d.py:721(WORLD)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3486(validate_elements)\n", - " 4 0.000 0.000 0.000 0.000 {method 'join' of 'str' objects}\n", - " 2 0.000 0.000 0.000 0.000 selectors.py:275(_key_from_fd)\n", - " 4 0.000 0.000 0.000 0.000 {method 'extend' of 'list' objects}\n", - " 1 0.000 0.000 0.000 0.000 jsonutil.py:38(_ensure_tzinfo)\n", - " 8 0.000 0.000 0.000 0.000 __init__.py:129(annotate)\n", - " 4 0.000 0.000 0.000 0.000 zmqstream.py:528(receiving)\n", - " 2 0.000 0.000 0.000 0.000 interactiveshell.py:1299(user_global_ns)\n", - " 1 0.000 0.000 0.000 0.000 {method 'write' of '_io.StringIO' objects}\n", - " 4 0.000 0.000 0.000 0.000 {method 'endswith' of 'str' objects}\n", - " 3 0.000 0.000 0.000 0.000 3480888288.py:38(__len__)\n", - " 4 0.000 0.000 0.000 0.000 pathlib.py:437(__str__)\n", - " 4 0.000 0.000 0.000 0.000 pickle.py:259(__init__)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method _contextvars.copy_context}\n", - " 2 0.000 0.000 0.000 0.000 {built-in method time.time}\n", - " 1 0.000 0.000 0.000 0.000 {method 'copy' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 typing.py:473(__repr__)\n", - " 3 0.000 0.000 0.000 0.000 typing.py:1227()\n", - " 4/2 0.000 0.000 0.000 0.000 {method 'acquire' of '_thread.lock' objects}\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:499(_index_sampler)\n", - " 2 0.000 0.000 0.000 0.000 threading.py:601(is_set)\n", - " 1 0.000 0.000 0.000 0.000 {method 'strip' of 'str' objects}\n", - " 1 0.000 0.000 0.000 0.000 {built-in method sys._getframe}\n", - " 1 0.000 0.000 0.000 0.000 {method '__enter__' of '_thread.lock' objects}\n", - " 1 0.000 0.000 0.000 0.000 {method 'get' of '_contextvars.ContextVar' objects}\n", - " 1 0.000 0.000 0.000 0.000 distributed_c10d.py:596(default_pg)\n", - " 1 0.000 0.000 0.000 0.000 queues.py:322(_consume_expired)\n", - " 2 0.000 0.000 0.000 0.000 interactiveshell.py:685(user_ns)\n", - " 1 0.000 0.000 0.000 0.000 iostream.py:255(closed)\n", - " 2 0.000 0.000 0.000 0.000 dataloader.py:495(_auto_collation)\n", - " 2 0.000 0.000 0.000 0.000 base_events.py:2003(get_debug)\n", - " 1 0.000 0.000 0.000 0.000 base_events.py:538(_check_closed)\n", - " 1 0.000 0.000 0.000 0.000 hmac.py:139(_current)\n", - " 1 0.000 0.000 0.000 0.000 displayhook.py:118(is_active)\n", - " 1 0.000 0.000 0.000 0.000 displaypub.py:150(is_publishing)\n", - " 1 0.000 0.000 0.000 0.000 history.py:1065(hold)\n", - " 1 0.000 0.000 0.000 0.000 :2(__init__)\n", - " 1 0.000 0.000 0.000 0.000 queues.py:59(_set_timeout)\n", - " 1 0.000 0.000 0.000 0.000 {method 'release' of '_thread.lock' objects}\n", - " 1/0 0.000 0.000 0.000 interactiveshell.py:3663(run_code)\n", - " 5/0 0.186 0.037 0.000 {built-in method select.select}\n", - " 1/0 0.000 0.000 0.000 {built-in method builtins.exec}\n", - "\n", - "\n" - ] - } - ], - "execution_count": 8 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:56.687464Z", - "start_time": "2025-08-03T13:32:56.488010Z" - } - }, - "cell_type": "code", - "source": [ - "profiler = cProfile.Profile()\n", - "profiler.enable()\n", - "iterate_train_loader() # Call the function to profile\n", - "profiler.disable()\n", - "profiler.print_stats(sort='cumulative') # Sort by cumulative time" - ], - "id": "758f62d0172b59d7", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 2375 function calls (2335 primitive calls) in 0.199 seconds\n", - "\n", - " Ordered by: cumulative time\n", - "\n", - " ncalls tottime percall cumtime percall filename:lineno(function)\n", - " 2 0.000 0.000 0.191 0.095 interactiveshell.py:3663(run_code)\n", - " 2 0.000 0.000 0.191 0.095 {built-in method builtins.exec}\n", - " 1 0.000 0.000 0.191 0.191 2121102005.py:1()\n", - " 8 0.011 0.001 0.165 0.021 dataloader.py:728(__next__)\n", - " 8 0.000 0.000 0.152 0.019 dataloader.py:787(_next_data)\n", - " 7 0.000 0.000 0.152 0.022 fetch.py:47(fetch)\n", - " 7 0.151 0.022 0.152 0.022 3480888288.py:75(__getitems__)\n", - " 1 0.026 0.026 0.031 0.031 2699206824.py:1(iterate_train_loader)\n", - " 2 0.007 0.004 0.007 0.004 {method '__exit__' of 'sqlite3.Connection' objects}\n", - " 97 0.000 0.000 0.001 0.000 3480888288.py:48(_load_single_sample)\n", - " 291 0.000 0.000 0.001 0.000 memmap.py:359(__getitem__)\n", - " 8 0.000 0.000 0.001 0.000 profiler.py:776(__exit__)\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:981(__call__)\n", - " 291 0.000 0.000 0.000 0.000 memmap.py:312(__array_finalize__)\n", - " 291 0.000 0.000 0.000 0.000 {built-in method torch.from_numpy}\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:1035(_must_dispatch_in_python)\n", - " 8 0.000 0.000 0.000 0.000 _pytree.py:1410(tree_any)\n", - " 8 0.000 0.000 0.000 0.000 profiler.py:770(__enter__)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method builtins.any}\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:1147(__call__)\n", - " 8 0.000 0.000 0.000 0.000 {built-in method torch._ops.profiler._record_function_enter_new}\n", - " 56/16 0.000 0.000 0.000 0.000 _pytree.py:1071(tree_iter)\n", - " 8 0.000 0.000 0.000 0.000 dataloader.py:722(_next_index)\n", - " 13 0.000 0.000 0.000 0.000 {built-in method builtins.next}\n", - " 8 0.000 0.000 0.000 0.000 sampler.py:326(__iter__)\n", - " 23 0.000 0.000 0.000 0.000 {built-in method torch.empty}\n", - " 8 0.000 0.000 0.000 0.000 {built-in method torch._ops.profiler.}\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:480(__iter__)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:419(_get_iterator)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:766(__init__)\n", - " 98 0.000 0.000 0.000 0.000 sampler.py:167(__iter__)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:634(__init__)\n", - " 2 0.000 0.000 0.000 0.000 events.py:86(_run)\n", - " 2 0.000 0.000 0.000 0.000 {method 'run' of '_contextvars.Context' objects}\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:708(__set__)\n", - " 32 0.000 0.000 0.000 0.000 _pytree.py:836(_is_leaf)\n", - " 56 0.000 0.000 0.000 0.000 _pytree.py:829(_get_node_type)\n", - " 2 0.000 0.000 0.000 0.000 ioloop.py:750(_run_callback)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3631(set)\n", - " 2 0.000 0.000 0.000 0.000 iostream.py:616(_flush)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:689(set)\n", - " 56 0.000 0.000 0.000 0.000 _pytree.py:818(_is_namedtuple_instance)\n", - " 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n", - " 2 0.000 0.000 0.000 0.000 iostream.py:710(_flush_buffers)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method torch.randperm}\n", - " 8 0.000 0.000 0.000 0.000 profiler.py:759(__init__)\n", - " 304 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n", - " 2 0.000 0.000 0.000 0.000 codeop.py:121(__call__)\n", - " 2 0.000 0.000 0.000 0.000 iostream.py:718(_rotate_buffers)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.compile}\n", - " 2 0.000 0.000 0.000 0.000 base_events.py:732(time)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:718(_validate)\n", - " 2 0.000 0.000 0.000 0.000 {method 'random_' of 'torch._C.TensorBase' objects}\n", - " 8 0.000 0.000 0.000 0.000 _pytree.py:603(_dict_flatten)\n", - " 291 0.000 0.000 0.000 0.000 multiarray.py:1418(may_share_memory)\n", - " 8 0.000 0.000 0.000 0.000 typing.py:378(inner)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method _heapq.heappop}\n", - " 8 0.000 0.000 0.000 0.000 _ops.py:1037()\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:1512(_notify_trait)\n", - " 18 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance}\n", - " 2 0.000 0.000 0.000 0.000 {method 'tolist' of 'torch._C.TensorBase' objects}\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3474(validate)\n", - " 2 0.000 0.000 0.000 0.000 {method 'item' of 'torch._C.TensorBase' objects}\n", - " 71 0.000 0.000 0.000 0.000 {built-in method builtins.len}\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:1523(notify_change)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:98(_get_distributed_settings)\n", - " 1 0.000 0.000 0.000 0.000 traitlets.py:1527(_notify_observers)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:727(_cross_validate)\n", - " 3 0.000 0.000 0.000 0.000 traitlets.py:676(__get__)\n", - " 16 0.000 0.000 0.000 0.000 _pytree.py:575(_tuple_flatten)\n", - " 3 0.000 0.000 0.000 0.000 contextlib.py:141(__exit__)\n", - " 1 0.000 0.000 0.000 0.000 events.py:127(__lt__)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3624(validate_elements)\n", - " 3 0.000 0.000 0.000 0.000 :117(__instancecheck__)\n", - " 2 0.000 0.000 0.000 0.000 contextlib.py:299(helper)\n", - " 1 0.000 0.000 0.000 0.000 __init__.py:14(is_available)\n", - " 15 0.000 0.000 0.000 0.000 {method 'values' of 'dict' objects}\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:75(create_fetcher)\n", - " 1 0.000 0.000 0.000 0.000 history.py:1016(_writeout_output_cache)\n", - " 1 0.000 0.000 0.000 0.000 distributed_c10d.py:1269(is_initialized)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method _thread.allocate_lock}\n", - " 3 0.000 0.000 0.000 0.000 {built-in method _abc._abc_instancecheck}\n", - " 2 0.000 0.000 0.000 0.000 contextlib.py:132(__enter__)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method time.monotonic}\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:2304(validate)\n", - " 2 0.000 0.000 0.000 0.000 contextlib.py:104(__init__)\n", - " 3 0.000 0.000 0.000 0.000 traitlets.py:629(get)\n", - " 1 0.000 0.000 0.000 0.000 {method 'manual_seed' of 'torch._C.Generator' objects}\n", - " 1 0.000 0.000 0.000 0.000 threading.py:299(__enter__)\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.max}\n", - " 8 0.000 0.000 0.000 0.000 {method '__exit__' of 'torch._C.DisableTorchFunctionSubclass' objects}\n", - " 1 0.000 0.000 0.000 0.000 selector_events.py:750(_process_events)\n", - " 2 0.000 0.000 0.000 0.000 sampler.py:160(num_samples)\n", - " 3 0.000 0.000 0.000 0.000 {method 'append' of 'collections.deque' objects}\n", - " 1 0.000 0.000 0.000 0.000 threading.py:314(_is_owned)\n", - " 4 0.000 0.000 0.000 0.000 compilerop.py:180(extra_flags)\n", - " 1 0.000 0.000 0.000 0.000 distributed_c10d.py:721(WORLD)\n", - " 1 0.000 0.000 0.000 0.000 {built-in method builtins.min}\n", - " 8 0.000 0.000 0.000 0.000 {method 'keys' of 'dict' objects}\n", - " 6 0.000 0.000 0.000 0.000 {method 'get' of 'dict' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'popleft' of 'collections.deque' objects}\n", - " 2 0.000 0.000 0.000 0.000 {built-in method builtins.iter}\n", - " 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n", - " 4 0.000 0.000 0.000 0.000 {built-in method builtins.getattr}\n", - " 7 0.000 0.000 0.000 0.000 3285181121.py:1(collate_fn)\n", - " 2 0.000 0.000 0.000 0.000 traitlets.py:3486(validate_elements)\n", - " 8 0.000 0.000 0.000 0.000 _jit_internal.py:101(is_scripting)\n", - " 2 0.000 0.000 0.000 0.000 interactiveshell.py:3615(compare)\n", - " 1 0.000 0.000 0.000 0.000 fetch.py:9(__init__)\n", - " 16 0.000 0.000 0.000 0.000 typing.py:2132(cast)\n", - " 1 0.000 0.000 0.000 0.000 threading.py:308(_release_save)\n", - " 2 0.000 0.000 0.000 0.000 {method '__exit__' of '_thread.lock' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'acquire' of '_thread.lock' objects}\n", - " 8 0.000 0.000 0.000 0.000 __init__.py:129(annotate)\n", - " 3 0.000 0.000 0.000 0.000 3480888288.py:38(__len__)\n", - " 2 0.000 0.000 0.000 0.000 interactiveshell.py:1299(user_global_ns)\n", - " 2 0.000 0.000 0.000 0.000 {method '__exit__' of '_thread.RLock' objects}\n", - " 2 0.000 0.000 0.000 0.000 {method 'extend' of 'list' objects}\n", - " 2 0.000 0.000 0.000 0.000 dataloader.py:495(_auto_collation)\n", - " 1 0.000 0.000 0.000 0.000 distributed_c10d.py:596(default_pg)\n", - " 1 0.000 0.000 0.000 0.000 dataloader.py:499(_index_sampler)\n", - " 1 0.000 0.000 0.000 0.000 history.py:1065(hold)\n", - " 1 0.000 0.000 0.000 0.000 {method '__enter__' of '_thread.lock' objects}\n", - " 2 0.000 0.000 0.000 0.000 interactiveshell.py:685(user_ns)\n", - " 1 0.000 0.000 0.000 0.000 {method 'release' of '_thread.lock' objects}\n", - "\n", - "\n" - ] - } - ], - "execution_count": 9 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:56.732003Z", - "start_time": "2025-08-03T13:32:56.695624Z" - } - }, - "cell_type": "code", - "source": "data = next(iter(train_loader))", - "id": "8a63394ca4331aa0", - "outputs": [], - "execution_count": 10 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:56.768913Z", - "start_time": "2025-08-03T13:32:56.763683Z" - } - }, - "cell_type": "code", - "source": "[(key, val.shape) for key, val in data.items()]", - "id": "5ca64ac63fb7566a", - "outputs": [ - { - "data": { - "text/plain": [ - "[('mhr', torch.Size([16, 8, 513, 256])),\n", - " ('ece', torch.Size([16, 48, 513, 256])),\n", - " ('co2', torch.Size([16, 4, 513, 256]))]" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "execution_count": 11 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-08-03T13:32:56.811605Z", - "start_time": "2025-08-03T13:32:56.808827Z" - } - }, - "cell_type": "code", - "source": "", - "id": "ef5a70e35399650d", - "outputs": [], - "execution_count": null - } - ], - "metadata": { - "kernelspec": { - "display_name": "ml-env [~/.conda/envs/ml-env/]", - "language": "python", - "name": "conda_ml-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/notebooks/test_model.ipynb b/scripts/notebooks/test_model.ipynb deleted file mode 100644 index 41e533c..0000000 --- a/scripts/notebooks/test_model.ipynb +++ /dev/null @@ -1,387 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 307, - "id": "09c9fef2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 308, - "id": "f3ccdbe3", - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "import argparse\n", - "import logging\n", - "\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "from torch.utils.data import ConcatDataset, DataLoader\n", - "\n", - "from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn\n", - "from tokamak_foundation_model.trainer.trainer import UnimodalTrainer\n", - "\n", - "from tokamak_foundation_model.models.modality import (\n", - " SlowTimeSeriesAutoEncoder,\n", - " FastTimeSeriesAutoEncoder,\n", - " SpatialProfileAutoEncoder,\n", - " SpectrogramAutoEncoder,\n", - " VideoAutoEncoder,\n", - ")\n", - "\n", - "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")" - ] - }, - { - "cell_type": "code", - "execution_count": 309, - "id": "c5f7829e", - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_REGISTRY = {\n", - " \"fast_time_series\": FastTimeSeriesAutoEncoder,\n", - " \"slow_time_series\": SlowTimeSeriesAutoEncoder,\n", - " \"profile\": SpatialProfileAutoEncoder,\n", - " \"spectrogram\": SpectrogramAutoEncoder,\n", - " \"video\": VideoAutoEncoder,\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 310, - "id": "68579454", - "metadata": {}, - "outputs": [], - "source": [ - "signal_name = 'ece'\n", - "model_name = 'spectrogram'\n", - "\n", - "d_model = 16\n", - "n_tokens = None\n", - "\n", - "working_dir = '/scratch/gpfs/nc1514/FusionAIHub/runs'\n", - "stats_path = '../../data/preprocessing_stats.pt'" - ] - }, - { - "cell_type": "code", - "execution_count": 311, - "id": "92b6d33c", - "metadata": {}, - "outputs": [], - "source": [ - "working_dir = Path(working_dir)\n", - "stats_path = Path(stats_path)\n", - "\n", - "checkpoint_path = working_dir / f\"{signal_name}_{model_name}\" / \"checkpoint.pth\"\n", - "data_dir = Path('/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data')" - ] - }, - { - "cell_type": "code", - "execution_count": 312, - "id": "9b736f3b", - "metadata": {}, - "outputs": [], - "source": [ - "stats = torch.load(stats_path)\n", - "\n", - "hdf5_files = sorted(data_dir.glob(\"*.h5\"))\n", - "datasets_processed = [\n", - " TokamakH5Dataset(\n", - " hdf5_path=str(f),\n", - " preprocessing_stats=stats,\n", - " input_signals=[signal_name],\n", - " target_signals=[signal_name],\n", - " prediction_mode=False,\n", - " )\n", - " for f in hdf5_files\n", - "]\n", - "\n", - "concatenated_dataset = ConcatDataset(datasets_processed)\n", - "\n", - "sample_data = next(iter(concatenated_dataset))[signal_name]" - ] - }, - { - "cell_type": "code", - "execution_count": 313, - "id": "f836ee84", - "metadata": {}, - "outputs": [], - "source": [ - "n_channels = sample_data.shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 314, - "id": "e917ac22", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SpectrogramAutoEncoder: n_channels=48, d_model=16, params=665,745\n" - ] - } - ], - "source": [ - "def build_model(model_name, n_channels, d_model, n_tokens):\n", - " \"\"\"Build the appropriate autoencoder.\n", - "\n", - " All autoencoders share the same interface: (n_channels, d_model, n_tokens).\n", - " \"\"\"\n", - " cls = MODEL_REGISTRY[model_name]\n", - " kwargs = dict(n_channels=n_channels, d_model=d_model)\n", - " if n_tokens is not None:\n", - " kwargs[\"n_tokens\"] = n_tokens\n", - " return cls(**kwargs)\n", - " \n", - "model = build_model(model_name, n_channels, d_model, n_tokens).to(device)" - ] - }, - { - "cell_type": "code", - "execution_count": 315, - "id": "a1c54033", - "metadata": {}, - "outputs": [], - "source": [ - "data = concatenated_dataset[5][signal_name]" - ] - }, - { - "cell_type": "code", - "execution_count": 316, - "id": "3ca33854", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Model loaded\n" - ] - } - ], - "source": [ - "model.load_state_dict(torch.load(checkpoint_path, weights_only=True))\n", - "model.eval()\n", - "print(\"Model loaded\")" - ] - }, - { - "cell_type": "code", - "execution_count": 317, - "id": "9ff26887", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABIMAAAGMCAYAAABJZWKFAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzsnXeYFFXa9u+e1JMHhpxBBJGgCCaCBEVQkFfUd112dUHk0zXn17ArYlxWXV3cXRVdVzAsZmUNqxhAMSCCBAEVAckyZGaYGSYwXd8f1U/X6erq7qqeznP/rquvma4+VXXOqVNVz7nPc57j0jRNAyGEEEIIIYQQQghpEmQkOgOEEEIIIYQQQgghJH5QDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCEkRsyZMwculwtz5sxJdFYIIYQQHxSDCCGEEJJwNm/eDJfLhUsuuSTRWSE24TUjhBBCUheKQYQQQgghhBBCCCFNCIpBhBBCCCGEEEIIIU0IikGEEELSikWLFmH8+PFo2bIl3G43evTogTvvvBPV1dVB00+YMAFt2rSB2+1Gp06dcP755+OLL77wS6dpGp599lkMGTIExcXFyM/Px4knnohnn302HsVyhMfjwTPPPIOTTz4ZpaWlyMvLQ8eOHTF+/Hh8+umnvnSffvopXC4X7r77bnzxxRcYMWIEioqK0KxZM1xwwQXYsGGD5fF3796NG2+8EUcffTTcbjdatmyJCy64AGvWrAma/uabb8YxxxyDvLw8lJaW4pRTTsFf/vIXAHpMlW7dugEAnnvuObhcLt9H8nv33Xf7vs+ZMwcDBgxAfn4+RowY4TvPli1bMHXqVHTo0AE5OTno2LEjpk6diq1bt1rm67vvvsPYsWNRVFSEkpISjB07FmvWrMEll1wCl8uFzZs3+9KqcV/eeecdDBkyBEVFRejatSsAoK6uDn//+98xZswYdOrUCW63G61bt8b555+PFStWBJzbfLxTTjkF+fn56NChA6ZNmwaPx+Orj+OPPx55eXno3LkzHn74YcuyWOE0T2odh8qvfA93zQCgqqoK06dPR69evZCbm4vS0lKMGzcOX375pWWendxnan7nzp2L/v37Iy8vD+3atcP111+Pw4cPW57D7j3vNO/79+/HFVdcgTZt2iA/Px8nnXQS3nrrLcu0wnfffYeJEyeiXbt2yMnJQZcuXXDttddi3759funUKXk//PADzjvvPLRo0SKgnRJCCCF2yUp0BgghhJBo8eSTT+Lqq69Gs2bNMH78eLRu3RrLli3DAw88gIULF2LhwoXIycnxpX/sscdw4403Ii8vD+eddx46d+6MHTt24IsvvsDrr7+OoUOHAtA7qBdddBFeeukl9OjRA7/97W+Rk5ODjz76CFOnTsX333/vEzaSgTvuuAMPPfQQunfvjt/+9rcoKirylevjjz/2E1AA4Ouvv8aMGTNw1lln4dprr8XatWvx1ltv4fPPP8fXX3+No446ypd248aNGDFiBLZv347Ro0djwoQJ2L17N9544w3Mnz8fn3zyCU455RRf+nXr1mHkyJHYuXMnhg4digkTJqCqqgpr167Fn/70J9xyyy3o378/rr/+ejz22GM4/vjjMWHCBN/+IrYIDz/8MBYuXIhzzz0Xo0ePRmZmJgDgp59+wtChQ7Fnzx6MHz8effr0wZo1a/Dss8/inXfewRdffIGePXv6jrNq1SqcdtppqKqqwvnnn48ePXpg2bJlGDp0KI4//vigdfvaa6/hww8/xDnnnIOrrroKFRUVAHQh4IYbbsBpp52GsWPHonnz5vj555/x9ttv4/3338eiRYtw0kknBRzvrbfewocffogJEyZgyJAheO+993D//fdD0zSUlJTg/vvvx7nnnosRI0bgjTfewK233oo2bdpg0qRJYdtBpHmyg51rVlNTg9NPPx3ffPMNBgwYgBtuuAG7du3CK6+8gvnz5+Oll17Cr371K99+kd5n//jHP/DBBx/g3HPPxemnn44PPvgAf/vb37B37178+9//9ktr9553mvfq6mqMGDECq1evxqBBgzB8+HBs27YNv/71rzF69GjLOnz77bdx4YUXIiMjA+eeey46deqE77//Hv/4xz8wf/58LFmyBM2bN/fbZ8OGDTj11FPRr18/XHLJJdi3b5/fM40QQgixjUYIIYSkAWvXrtWysrK0448/Xtu7d6/fbzNmzNAAaH/5y19821auXKllZGRo7du31zZt2uSX3uPxaDt27PB9f/rppzUA2pQpU7S6ujrf9traWm38+PEaAG3ZsmWxKVgElJaWau3bt9eqqqoCftu3b5/v/4ULF2oANADarFmz/NLNmjVLA6Cdc845ftsHDx6sZWZmah988IHf9nXr1mlFRUVav379/LafeOKJGgDt6aefDsjLtm3bfP9v2rRJA6BNnjzZskzTp0/XAGgFBQXad999F/D7yJEjNQDaU0895bf98ccf1wBop59+ut/2oUOHagC0f//7337bp02b5qsTtV3Mnj1bA6BlZGRoH330UcD5a2pqtO3btwdsX7NmjVZYWKiNGjXKb7scLzs7W/vmm2982ysqKrTWrVtr+fn5Wtu2bbWNGzf6ftu6dauWk5MTUMfBcJonqeOFCxcG7CP5nT17tm9buGt2zz33aAC0iy66SPN4PL7ty5cv13JycrRmzZppFRUVvu1O7zPJb0lJifbjjz/6tldXV2s9e/bUMjIy/O5jJ/e807xLXi677DK/437wwQe+9qTW3d69e7Xi4mKtQ4cO2ubNm/32eemllzQA2jXXXOPbJnUNQLvrrrsCK5sQQghxCMUgQgghacF1112nAdAWLVoU8FtDQ4PWqlUrbeDAgb5tV155pQZAe/bZZ8Me+7jjjtMKCgq06urqgN++++47DYB28803N64AUaS0tFTr2rWrVlNTEzKdiEE9e/bUGhoa/H5raGjQevTooblcLm337t2apukdYQDapZdeanm8m266SQOgrV69WtM0TVuyZIkGQBs2bFjYPNsVg2688caA37Zs2aIB0Hr37u3XcZdy9OrVSwOgbd26VdM0Tdu8ebMGQDv++OMDjlVZWak1b948qBh03nnnhS2LmfHjx2s5OTl+Aoccb8qUKQHpL730Ug2Ads899wT8dvrpp2uZmZlafX2943yEy1O0xaCjjjpKy87O9hP9hMsuu0wDoD3//PO+bU7vM8mvlTgiv7399tu+bU7uead579atm5aTk6Pt3LkzIP0ZZ5wRUHePPvpowDFUBgwYoLVs2dL3Xeq6bdu2Wm1tbdj8E0IIIeHgNDFCCCFpwddffw0AvqlKZrKzs/Hjjz/6vn/zzTcAEHQKh1BdXY3Vq1ejffv2ePDBBwN+r6+vBwC/Ywfj7rvvDpsmHP379/ebkmPFxIkT8cQTT6Bv376YOHEiRo4ciUGDBiEvL88y/ZAhQ5CR4R9GMCMjA0OGDMH69euxatUqjBo1ylfHu3btsiyL1MGPP/6Ivn372q5jJ5x88skB21auXAkAGD58OFwuV0A5hg0bhh9//BErV65Ep06dsGrVKgB6uc0UFBSgf//+WLhwoe3zq/l46KGH8MUXX6CsrMzXNoS9e/eiXbt2ftv69+8fcBxJE+y3hoYG7Nq1Cx06dAial8bkKRpUVFTg559/xrHHHouOHTsG/D5y5Ej885//xMqVK/G73/2uUffZwIEDA7bJOQ8ePOjbZrc9Os17RUUFNm3ahN69e6Nt27YB6U877bSAZ5LcS0uWLMHGjRsD9qmpqcHevXuxd+9etGzZ0rf9+OOP57QwQgghUYFiECGEkLRg//79AIAHHnjAVvry8nK4XK6wHeEDBw5A0zTs2LED99xzT9B0VVVVYc8Zan+7TJ48OawY9Nhjj6Fbt26YPXs27r//ftx///3Izc3FhRdeiEceecSvcwkAbdq0sTyObC8vLwdg1PF7772H9957L+j5pS5kPzuihV2s8ipxe4KVQ66xpJO/rVu3tn2OcL999dVXOP300wHoYkOPHj1QWFgIl8uFefPmYdWqVaitrQ3Yr7i4OGBbVlZW2N/Mok408xQNnF6TxtxnoeqpoaHBt83uPR+P9iT30uOPPx4yL1VVVX73a6i2SQghhDiBYhAhhJC0QDqEFRUVKCoqCpu+WbNm0DQNO3fuDClWyHEHDhyIZcuWNSqPmqY1an+7ZGVl4ZZbbsEtt9yCX375BZ999hlmz56N559/HmVlZZg/f75f+l27dlkeR7aXlJQAMOri73//O6655pqw+WjWrBkAYMeOHZEWJQCz54+ar2DlKCsr80snf3fv3m2ZPthxgp0f0EXI2tpafP75574gxMLXX3/t80aKJ07zJN5hR44cCTiWCHt2ifSaROM+C4bTez6W7Un2Wb16Nfr27WuzBMHbHyGEEOIULi1PCCEkLZAVrGT6RThkus+HH34YMl1RURGOPfZY/PDDD35TTlKF9u3b4ze/+Q0++OADHH300fj4448Dltz+8ssvfUuZCx6PB1999RVcLpdvdS2p48WLF9s6t906BuBbFUz15LCLTKdatGhRgOCmaRoWLVrkl07K89VXXwUcq7q6OiLhZuPGjSgtLQ0QXaqrq7F8+XLHx4sGTvMkK1dZiXdWS9GHumbFxcU46qijsGHDBsvjyfLzck3icZ/ZbY9O815cXIxu3bphw4YNPqFI5fPPPw/Y5vReIoQQQqINxSBCCCFpwVVXXYWsrCxce+212Lp1a8DvBw8e9OvQXnHFFcjMzMSdd96JLVu2+KXVNA2//PKL7/t1112H6upqXHbZZZbTVDZt2oTNmzdHrzCNoLa21lLkqKqqQmVlJbKzswPiA/3000/45z//6bftn//8J3766SeMGzcOrVq1AqB3pk855RS89NJLeOWVVwLO4fF48Nlnn/m+n3TSSTjppJOwaNGigOMD/qJD8+bN4XK5sG3bNmcFBtC5c2eMHDkSa9euxbPPPuv329NPP40ffvgBp59+Ojp16gQA6NKlC4YMGYKVK1cGlOPhhx/2TeFxQpcuXXDgwAGsXbvWt62hoQG33HIL9uzZ4/h40cBpnmSZ+eeff95PHFy8eHHAEu1A+Gs2efJk1NfX44477vAT6b777jvMmTMHJSUlflMeY32fObnnneb9d7/7Herq6nDXXXf5HffDDz+0jGE2ZcoUFBUV4Y9//KPf9RGqq6ttC9uEEEJIJHCaGCGEkLSgb9++eOKJJ3DllVfimGOOwdixY9G9e3ccOnQIP//8Mz777DNccsklmDVrFgCgX79+mDlzJq677jr06dMHEyZMQJcuXVBWVoZFixZh3LhxmDlzJgDg97//Pb7++ms899xz+PLLLzFq1Ci0b98eu3btwo8//oglS5Zg7ty56Nq1a+IqwMvhw4cxZMgQ9OzZEwMHDkTnzp1RWVmJd999F2VlZbjlllvgdrv99hkzZgyuu+46/Pe//0WfPn2wdu1avPPOO2jZsiUee+wxv7QvvfQSRo4ciYkTJ2LmzJkYMGAA8vLysHXrVixevBh79uxBTU2NL/2///1vjBgxApdffjleeOEFDBo0CDU1NVi7di1WrFiBffv2AQAKCwt9wtHvfvc79OjRAxkZGfjd736HLl26hC33k08+iaFDh+Kyyy7DO++8g969e2Pt2rV4++230apVKzz55JN+6f/+979j2LBhuOiii/DGG2/g6KOPxvLly/H1119j2LBhWLRoUYBoFoprr70WH374IYYOHYoLL7wQubm5+PTTT7Fjxw6MGDHC500ST5zm6dRTT8WQIUOwYMECDBo0CMOGDcOWLVvwn//8B+PHj8dbb73llz7cNbv11lvx3nvv4YUXXsAPP/yAM844A7t378Yrr7yCI0eO4J///KfflM5Y32dO7nmneb/11lvx5ptv4p///CfWrl2LYcOGYdu2bXj11Vcxbty4gBhbrVq1wksvvYRf/epXOP7443HWWWehV69eqK2txebNm/HZZ59h8ODB+OCDDyIqKyGEEBKWBK1iRgghhMSEb775Rps4caLWvn17LTs7W2vZsqU2YMAA7fbbb9d++OGHgPQLFy7UzjnnHK20tFTLycnROnbsqF1wwQXal19+GZD2lVde0UaNGqU1b95cy87O1jp06KCNGDFCe+SRR7Q9e/bEo3hhqaur0x588EFt9OjRWseOHbWcnBytTZs22rBhw7S5c+f6Lb0uS8tPnz5d+/zzz7Xhw4drBQUFWnFxsXbeeedp69evtzzH/v37tTvvvFPr27evlpeXpxUWFmo9evTQfvvb32pvvvlmQPqysjLt+uuv14466igtJydHKy0t1U455RTt0Ucf9Uu3bt06bezYsVqzZs00l8vlt8x5qGXPhc2bN2tTpkzR2rVrp2VlZWnt2rXTpkyZom3evNky/YoVK7QxY8ZohYWFWlFRkXb22Wdrq1ev1s455xwNgHbgwAFfWqul1c28/vrr2oABA7T8/HytZcuW2oUXXqht3LhRmzx5ctCl6q2OF6qsVscKhZM8aZqm7d27V5s0aZJWWlqq5eXlaaeeeqo2f/78oPkNdc00TdMqKyu1adOmaT179tRycnK0Zs2aaWeffbb2+eefB82z3fssVD2Fql+797zTvO/bt0+7/PLLtVatWmm5ubnawIEDtTfffDNkXn788Udt6tSpWpcuXbScnBytefPmWr9+/bTrrrtO++abb3zpZGn5yZMnB603QgghxAkuTYtTNEtCCCGEJBWffvopRo4cienTp0dl2ft0oKGhAd27d8fhw4dDBpImhBBCCEllGDOIEEIIIU2OI0eOYO/evQHb//znP2PLli1+8WAIIYQQQtINxgwihBBCSJOjsrISHTp0wJlnnomePXuivr4eS5YswdKlS9GuXTt6ShFCCCEkraEYRAghhJAmR35+PqZOnYoFCxZg0aJFqKmpQbt27fD73/8e06ZNQ7t27RKdRUIIIYSQmMGYQYQQQgghhBBCCCFNCMYMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIoQQQgghhBBCCGlCUAwihBBCCCGEEEIIaUJQDCKEEEIIIYQQQghpQlAMIqQJ8eqrr6K0tBSVlZWO9uvatSvOOeecGOUq/nTt2hWXXHJJ3M9bX1+PTp064Yknnoj7uQkhhBASOZ9++ilcLhc+/fTThJzf4/Ggb9++eOCBBxztN2fOHLhcLixbtixGOYsvUp7NmzfH/dyzZs1C586dUVtbG/dzExILKAYRYoNke5FWV1fj7rvvdmSQNDQ0YPr06bj22mtRWFgYu8ylEa+88gouvvhi9OjRAy6XCyNGjAia9ttvv8VZZ52F4uJiFBUVYfTo0Vi5cqVfmuzsbNx000144IEHUFNTE9vME0IISXnE/pBPVlYWOnTogEsuuQQ7duxIdPaizhNPPIE5c+Y0+TxY8dJLL2Hbtm245pprEp2VlGDnzp24/fbbMXLkSBQVFYUU8urr63HPPffgqKOOgtvtxlFHHYX7778fR44c8Ut3ySWXoK6uDk899VQcSkBI7KEYREgKUl1djXvuuceRGPTOO+9g3bp1uPzyy2OXsTTjySefxH/+8x906tQJzZs3D5pu+fLlGDp0KH7++WdMnz4dd911F9avX4/hw4dj3bp1fmmnTJmCvXv3Yu7cubHOPiGEkDTh3nvvxQsvvIBZs2bh7LPPxosvvojhw4en3cBCMggxwfIwbNgwHD58GMOGDYt/pgA8/PDDmDhxIkpKShJy/lRj3bp1ePDBB7Fjxw7069cvZNqLL74Y99xzD04//XQ89thjGDZsGKZNm4arrrrKL11ubi4mT56MRx99FJqmxTL7hMQFikGENBFmz56NIUOGoEOHDonOSsrwwgsvoLy8HAsWLED79u2Dpps2bRry8vKwePFi3Hzzzfi///s/fPXVV/B4PPjDH/7gl7ZZs2YYPXp0wo1dQgghqcPZZ5+Niy++GP/v//0/PPPMM7jllluwceNGvP3224nOWsKoqqqK6/kyMjKQm5uLjIz4d59WrFiBVatW4cILL4z7uVOVgQMHYt++ffjpp59w0003BU23dOlSvPrqq7jzzjvxzDPP4IorrsCcOXNw880345lnnsF3333nl/7CCy/Eli1bsHDhwlgXgZCYQzGIkAi55JJLUFhYiB07dmDChAkoLCxEq1atcMstt6ChocGXbvPmzXC5XPjLX/6Cv/71r+jSpQvy8vIwfPhwrFmzxu+YI0aMsJyKdMkll6Br166+47Vq1QoAcM899/hcx+++++6gea2pqcEHH3yAUaNGWf7+4osv4uSTT0Z+fj6aN2+OYcOG4cMPPwxI98UXX+Dkk09Gbm4ujjrqKDz//PN+v+/fvx+33HIL+vXrh8LCQhQXF+Pss8/GqlWr/NLJvPtXX30VDzzwADp27Ijc3FycccYZ2LBhQ0Cd9O3bF99//z1GjhyJ/Px8dOjQAQ899FBA/mprazF9+nQcffTRcLvd6NSpE2699daI53Z36tTJltH3+eefY9SoUWjRooVvW7t27TB8+HC8++67ATGazjzzTHzxxRfYv39/RPkihBDStDnttNMAABs3bvTb/uOPP+J///d/UVpaitzcXJx44omWgtHBgwdx4403omvXrnC73ejYsSMmTZqEvXv3+tLs3r0bU6dORZs2bZCbm4vjjz8ezz33nN9xVBvn6aefRvfu3eF2u3HSSSdh6dKlfmnLysowZcoUdOzYEW63G+3atcO5557ri/3StWtXrF27Fp999pnPthGbSKbLffbZZ7jqqqvQunVrdOzYEYC/jaRy9913w+VyBWwPZfOEykOwmEGvvfYaBg4ciLy8PLRs2RIXX3xxwBQ+uzZjMObNm4ecnBxLr6QdO3Zg6tSpaN++PdxuN7p164Yrr7wSdXV1fulqa2tx0003oVWrVigoKMB5552HPXv2+KX5z3/+g3HjxvmO1b17d9x3330BebRrmzmx9wBgyZIlOOuss1BSUoL8/HwMHz4cX375Zdj6saKoqAilpaVh033++ecAgIkTJ/ptnzhxIjRNwyuvvOK3feDAgSgtLcV//vOfiPJFSDKRlegMEJLKNDQ0YMyYMTjllFPwl7/8BR9//DEeeeQRdO/eHVdeeaVf2ueffx6HDh3C1VdfjZqaGjz22GM4/fTTsXr1arRp08b2OVu1aoUnn3wSV155Jc477zycf/75AIDjjjsu6D7ffvst6urqMGDAgIDf7rnnHtx9990YPHgw7r33XuTk5GDJkiVYsGABRo8e7Uu3YcMG/O///i+mTp2KyZMn49lnn8Ull1yCgQMHok+fPgCAn3/+GfPmzcOvfvUrdOvWDbt27cJTTz2F4cOH4/vvvw/wrvnzn/+MjIwM3HLLLSgvL8dDDz2Eiy66CEuWLPFLd+DAAZx11lk4//zzceGFF+L111/Hbbfdhn79+uHss88GoAdW/J//+R988cUXuPzyy3Hsscdi9erV+Otf/4qffvoJ8+bNs13HTqmtrUVeXl7A9vz8fNTV1WHNmjU49dRTfdsHDhwITdPw1VdfpVVgbkIIIfFBBBR1CvPatWt9HsC33347CgoK8Oqrr2LChAl44403cN555wEAKisrcdppp+GHH37ApZdeigEDBmDv3r14++23sX37drRs2RKHDx/GiBEjsGHDBlxzzTXo1q0bXnvtNVxyySU4ePAgrr/+er/8zJ07F4cOHcLvf/97uFwuPPTQQzj//PPx888/Izs7GwBwwQUXYO3atbj22mvRtWtX7N69Gx999BG2bt2Krl27YubMmb64hn/84x8BIMA+uuqqq9CqVSvcddddEXkGhbN57ORBZc6cOZgyZQpOOukkzJgxA7t27cJjjz2GL7/8EitWrECzZs18aZ3YjGa++uor9O3b11eXwi+//IKTTz4ZBw8exOWXX45evXphx44deP3111FdXY2cnBxf2muvvRbNmzfH9OnTsXnzZsycORPXXHONn9gxZ84cFBYW4qabbkJhYSEWLFiAu+66CxUVFXj44Yf9zm3HNhPs2HsLFizA2WefjYEDB2L69OnIyMjA7Nmzcfrpp+Pzzz/HySefHLKOIkUGDM12XH5+PgDdhjYzYMCAiEUqQpIKjRASltmzZ2sAtKVLl/q2TZ48WQOg3XvvvX5pTzjhBG3gwIG+75s2bdIAaHl5edr27dt925csWaIB0G688UbftuHDh2vDhw8POP/kyZO1Ll26+L7v2bNHA6BNnz7dVv6feeYZDYC2evVqv+3r16/XMjIytPPOO09raGjw+83j8fj+79KliwZAW7RokW/b7t27Nbfbrd18882+bTU1NQHH2bRpk+Z2u/3qaeHChRoA7dhjj9Vqa2t92x977LGAfA4fPlwDoD3//PO+bbW1tVrbtm21Cy64wLfthRde0DIyMrTPP//c7/yzZs3SAGhffvmlX3kmT55sXVlB6NOnj+W10TRN69evn9azZ0/tyJEjfnns3LmzBkB7/fXX/dL/8ssvGgDtwQcfdJQHQgghTQuxPz7++GNtz5492rZt27TXX39da9WqleZ2u7Vt27b50p5xxhlav379tJqaGt82j8ejDR48WOvRo4dv21133aUB0N58882A88m7f+bMmRoA7cUXX/T9VldXpw0aNEgrLCzUKioqNE0zbJwWLVpo+/fv96X9z3/+owHQ3nnnHU3TNO3AgQMaAO3hhx8OWd5g71qph6FDh/q9azUt0EYSpk+frqldHbs2T7A8iO2ycOFCX320bt1a69u3r3b48GFfunfffVcDoN11111+ebRjMwajY8eOfjaPMGnSJC0jI8PPPjWXSepu1KhRfuW88cYbtczMTO3gwYO+bdXV1QHH+f3vf6/l5+f7tSu7tplde8/j8Wg9evTQxowZ45fH6upqrVu3btqZZ57p2ybl2bRpk3VlWfDaa6/5XTuVN954QwOgvfDCC37bxX7s27dvwD6XX365lpeXZ/v8hCQrnCZGSCO54oor/L6fdtpp+PnnnwPSTZgwwS9ez8knn4xTTjkF//3vf2Oex3379gFAQBDkefPmwePx4K677gqYDmV2re7du7fPLR3QPZSOOeYYv7K63W7fcRoaGrBv3z4UFhbimGOOwfLlywPyNWXKFL9RKzm+uf4KCwtx8cUX+77n5OTg5JNP9kv32muv4dhjj0WvXr2wd+9e3+f0008HgJjO7b7qqqvw008/YerUqfj++++xZs0aTJo0CTt37gQAHD582C+9XAfVHZ8QQggJxqhRo9CqVSt06tQJ//u//4uCggK8/fbbvqlS+/fvx4IFC3DhhRfi0KFDvnfgvn37MGbMGKxfv943demNN97A8ccf7/MUUpF3/3//+1+0bdsWv/nNb3y/ZWdn47rrrkNlZSU+++wzv/1+/etf+9kY5vd5Xl4ecnJy8Omnn+LAgQMR18Nll12GzMzMiPZ1YvPYYdmyZdi9ezeuuuoq5Obm+raPGzcOvXr1wnvvvRewj12b0cy+ffsCbDiPx4N58+Zh/PjxOPHEEwP2MZfp8ssv99t22mmnoaGhAVu2bPFtU71jpB2ddtppqK6uxo8//uh3PDu2mRDO3lu5ciXWr1+P3/72t9i3b5+v/VZVVeGMM87AokWL4PF4gldQIxg7diy6dOmCW265BW+++Sa2bNmCV199FX/84x+RlZUVYMMBuh13+PBhVFdXxyRPhMQLikGENILc3Fxf/B6hefPmloZOjx49Arb17NnT5+odDzTTygcbN25ERkYGevfuHXbfzp07B2wzl9Xj8eCvf/0revToAbfbjZYtW6JVq1b47rvvUF5eHvaYYuiY669jx44BRo353OvXr8fatWvRqlUrv0/Pnj0B6LEPYsUVV1yBP/zhD5g7dy769OmDfv36YePGjbj11lsB6AaTilyHSIxPQgghTY/HH38cH330EV5//XWMHTsWe/fuhdvt9v2+YcMGaJqGadOmBbwHp0+fDsB4D27cuBF9+/YNeb4tW7agR48eAaLJscce6/tdJdz73O1248EHH8T777+PNm3aYNiwYXjooYdQVlbmqB66devmKL2KE5vHDlIHxxxzTMBvvXr1CqgjJzajFWYbbs+ePaioqAh7LQU7NtfatWtx3nnnoaSkBMXFxWjVqpVP8DHbcXZsM7vnXr9+PQBg8uTJAe33mWeeQW1traUdGQ1yc3Px3nvvoUWLFrjgggvQtWtXTJo0CXfddRdKS0sDbDiAdhxJHxgziJBGEOnoVDBcLpflUpV2gguGQgIbHzhwwDeK6JRgZVXz+6c//QnTpk3DpZdeivvuuw+lpaXIyMjADTfcYDmiY+eYdtN5PB7069cPjz76qGXaTp06WW6PFg888ABuueUWrF27FiUlJejXr59vJTERpAQxflq2bBnTPBFCCEkPTj75ZJ/3x4QJEzB06FD89re/xbp161BYWOh7x95yyy0YM2aM5TGOPvromOXPznv6hhtuwPjx4zFv3jzMnz8f06ZNw4wZM7BgwQKccMIJts5jFZ8vWIe8sbZTtGmMzdiiRYtGeVSFOr9co4MHD2L48OEoLi7Gvffei+7duyM3NxfLly/HbbfdFmDH2bXh7KSVYz/88MPo37+/ZVorUSZa9OnTB2vWrMH333+PAwcOoHfv3sjLy8ONN96I4cOHB6Q/cOAA8vPzLdsjIakExSBC4oSMeqj89NNPfitgNG/e3NK91jy65HQkolevXgCATZs2oV+/fr7t3bt3h8fjwffffx/05euE119/HSNHjsS//vUvv+0HDx6MufDRvXt3rFq1CmeccUbCRmqaN2+OoUOH+r5//PHH6Nixo6/+hU2bNgEwRlgJIYQQu2RmZmLGjBkYOXIk/vGPf+D222/HUUcdBUCfyhVs5VChe/fuAauZmunSpQu+++47eDweP+8gmSrUpUuXiPLevXt33Hzzzbj55puxfv169O/fH4888ghefPFFAJF5WjRv3hwHDx4M2G62nezaPHbzIHWwbt0635R0Yd26dRHXkRW9evXy2Q5Cq1atUFxcHPZa2uXTTz/Fvn378Oabb/qtWmY+byzo3r07AKC4uDhs+40VLpfLtyAKoE+V9Hg8lvnZtGkTbTiSFnCaGCFxYt68eX5LjX7zzTdYsmSJ34oL3bt3x48//ui31OeqVasCViyQFQ6sjB8rBg4ciJycHCxbtsxv+4QJE5CRkYF77703YMTHamQnHJmZmQH7vfbaawFLrMaCCy+8EDt27MA///nPgN8OHz4c0aojjeGVV17B0qVLccMNNwS42X/77bdwuVwYNGhQXPNECCEkPRgxYgROPvlkzJw5EzU1NWjdujVGjBiBp556yhevTkW1Ky644AKsWrUKb731VkA6eYePHTsWZWVlfitNHTlyBH//+99RWFho6S0RiurqatTU1Pht6969O4qKinyrOQFAQUGBbdtGPU55eTm+++4737adO3cGlM+uzWM3DyeeeCJat26NWbNm+ZXh/fffxw8//IBx48Y5KkcoBg0ahDVr1vidJyMjAxMmTMA777wTYN8Bzu048d5R96urq8MTTzwRYa7tM3DgQHTv3h1/+ctfUFlZGfC72n7jweHDhzFt2jS0a9fOL26WsHz5cgwePDiueSIkFtAziJA4cfTRR2Po0KG48sorUVtbi5kzZ6JFixa+uDIAcOmll+LRRx/FmDFjMHXqVOzevRuzZs1Cnz59UFFR4UuXl5eH3r1745VXXkHPnj1RWlqKvn37Bp03npubi9GjR+Pjjz/Gvffe65enP/7xj7jvvvtw2mmn4fzzz4fb7cbSpUvRvn17zJgxw1EZzznnHNx7772YMmUKBg8ejNWrV+Pf//63b8Qylvzud7/Dq6++iiuuuAILFy7EkCFD0NDQgB9//BGvvvoq5s+fbxlgMRSLFi3CokWLAOiGSFVVFe6//34AwLBhw3wjZ4sWLcK9996L0aNHo0WLFvj6668xe/ZsnHXWWQHL7wLARx99hCFDhvim7xFCCCFO+b//+z/86le/wpw5c3DFFVfg8ccfx9ChQ9GvXz9cdtllOOqoo7Br1y4sXrwY27dvx6pVq3z7vf766/jVr36FSy+9FAMHDsT+/fvx9ttvY9asWTj++ONx+eWX46mnnsIll1yCb7/9Fl27dsXrr7+OL7/8EjNnzkRRUZGjvP70008444wzcOGFF6J3797IysrCW2+9hV27dmHixIm+dAMHDsSTTz6J+++/H0cffTRat24d4HVjZuLEibjttttw3nnn4brrrkN1dTWefPJJ9OzZ02/xCrs2j908ZGdn48EHH8SUKVMwfPhw/OY3v/EtLd+1a1fceOONjuooFOeeey7uu+8+fPbZZxg9erRv+5/+9Cd8+OGHGD58OC6//HIce+yx2LlzJ1577TV88cUXfkvbh2Pw4MFo3rw5Jk+ejOuuuw4ulwsvvPBCRIODTsnIyMAzzzyDs88+G3369MGUKVPQoUMH7NixAwsXLkRxcTHeeecdx8cVm23t2rUAgBdeeAFffPEFAODOO+/0pbvwwgvRvn179O7dGxUVFXj22Wfx888/47333gto699++y3279+Pc889N9LiEpI8xHfxMkJSk2BLyxcUFASkNS9lKsuuPvzww9ojjzyiderUSXO73dppp52mrVq1KmD/F198UTvqqKO0nJwcrX///tr8+fMtl0396quvtIEDB2o5OTm2lpl/8803NZfLpW3dujXgt2effVY74YQTNLfbrTVv3lwbPny49tFHH/l+79KlizZu3LiA/YYPH+63/GpNTY128803a+3atdPy8vK0IUOGaIsXLw5IJ0uNvvbaa37Hk7qaPXu23zn69OkTcG6rOqmrq9MefPBBrU+fPr6yDBw4ULvnnnu08vJyv/LYWVperqXVR63vDRs2aKNHj9Zatmypud1urVevXtqMGTP8llEVDh48qOXk5GjPPPNM2PMTQghp2ljZH0JDQ4PWvXt3rXv37r7l1jdu3KhNmjRJa9u2rZadna116NBBO+ecc7TXX3/db999+/Zp11xzjdahQwctJydH69ixozZ58mRt7969vjS7du3SpkyZorVs2VLLycnR+vXr5/d+1jR/G8eM+q7cu3evdvXVV2u9evXSCgoKtJKSEu2UU07RXn31Vb99ysrKtHHjxmlFRUUaAJ/tEKoeNE3TPvzwQ61v375aTk6Odswxx2gvvvhigD0mhLN5guXBvLS88Morr/iOV1paql100UXa9u3b/dLYtRlDcdxxx2lTp04N2L5lyxZt0qRJWqtWrTS3260dddRR2tVXX+2zQYLVnVV5vvzyS+3UU0/V8vLytPbt22u33nqrNn/+/IB0dm0zJ/aepmnaihUrtPPPP19r0aKF5na7tS5dumgXXnih9sknn/jSOFlaPpgNZ67zBx98UOvVq5eWm5urNW/eXPuf//kfbcWKFZbHvO2227TOnTtrHo8n7PkJSXZcmhYHuZeQJszmzZvRrVs3PPzww7jlllsSlo+Ghgb07t0bF154Ie67776E5aOpM3PmTDz00EPYuHEjAw8SQgghxBYvvPACrr76amzdutWRxw+JHrW1tejatStuv/12S89vQlINxgwipImQmZmJe++9F48//rjlfGwSe+rr6/Hoo4/izjvvpBBECCGEENtcdNFF6Ny5Mx5//PFEZ6XJMnv2bGRnZ+OKK65IdFYIiQr0DCIkxiSLZxAhhBBCCCGEEALQM4gQQgghhBBCCCGkSUExiJAY07VrV2iaRq8gQghR+POf/wyXy4UbbrghZLrXXnsNvXr1Qm5uLvr164f//ve/8ckgIYQQQkiciad9RDGIEEIIIXFl6dKleOqpp3DccceFTPfVV1/hN7/5DaZOnYoVK1ZgwoQJmDBhAtasWROnnBJCCCGExId420eMGQTA4/Hgl19+QVFREVwuV6KzQwghhAAANE3DoUOH0L59e2RkxHb8pqamBnV1dY730zQt4N3pdrvhdrst01dWVmLAgAF44okncP/996N///6YOXOmZdpf//rXqKqqwrvvvuvbduqpp6J///6YNWuW47wSZ9A+IoQQkozE0z4CIrORUsE+yrKdMo355Zdf0KlTp0RngxBCCLFk27Zt6NixY8yOX1NTg27duqGsrMzxvoWFhQErFE6fPh133323Zfqrr74a48aNw6hRo3D//feHPPbixYtx0003+W0bM2YM5s2b5zifxDm0jwghhCQzsbaPgMhtpFSwjygGASgqKkp0FgghhJCgxPo9VVdXh7KyMmzbsgnFxcW296uoqECnLt2wbds2v/2CjXq9/PLLWL58OZYuXWrr+GVlZWjTpo3ftjZt2kQkWhHn0D4ihBCSzMTjPRWJjZQq9hHFIICuz4QQQpKaeL2nigvzUVyYb38HzxF9v+LisAbStm3bcP311+Ojjz5Cbm5uY7JJ4gTtI0IIIclMPN9TjmykFLGPKAYRQgghJOZ8++232L17NwYMGODb1tDQgEWLFuEf//gHamtrkZmZ6bdP27ZtsWvXLr9tu3btQtu2beOSZ0IIIYSQWJJI+4iriRFCCCFEx3PE+ccmZ5xxBlavXo2VK1f6PieeeCIuuugirFy5MsDQAYBBgwbhk08+8dv20UcfYdCgQY0uKiGEEEKIbdLQPqJnECGEEEJ0HBowTtIWFRWhb9++ftsKCgrQokUL3/ZJkyahQ4cOmDFjBgDg+uuvx/Dhw/HII49g3LhxePnll7Fs2TI8/fTT9vNICCGEENJYnNhIKWIf0TOIEEIIITqeBocjXw1RPf3WrVuxc+dO3/fBgwdj7ty5ePrpp3H88cfj9ddfx7x58wKMJkIIIYSQmOLIRkoN+8ilaZoW1ZymIBUVFSgpKUl0NgghhBBLysvLHa3y5RR5D5bv+AnFxfZX5qioOISSDj1jnj+SGGgfEUIISWbiYX9EYiOlin3EaWKEEEII0YnhNDFCCCGEkJQlRtPEEgnFIEIIIYToUAwihBBCCAmEYhAhhBBC0haKQYQQQgghgVAMIoQQQkjaojU4M2C06AZIJIQQQghJSpzYSCliH1EMIoQQQogOPYMIIYQQQgJJQ8+ghC4tv2jRIowfPx7t27eHy+XCvHnzfL/V19fjtttuQ79+/VBQUID27dtj0qRJ+OWXX/yOsX//flx00UUoLi5Gs2bNMHXqVFRWVsa5JIQQQkga4GhZeYfCEbEN7SNCCCEkyUhD+yihYlBVVRWOP/54PP744wG/VVdXY/ny5Zg2bRqWL1+ON998E+vWrcP//M//+KW76KKLsHbtWnz00Ud49913sWjRIlx++eXxKgIhhBCSPlAMSgpoHxFCCCFJRhraRy5N07REZwIAXC4X3nrrLUyYMCFomqVLl+Lkk0/Gli1b0LlzZ/zwww/o3bs3li5dihNPPBEA8MEHH2Ds2LHYvn072rdvb3mc2tpa1NbW+r5XVFSgU6dOUS0PIYQQEi3Ky8tRXFwcs+NXVFSgpKQE5d9/jOKiAvv7HapCSe9RMc9fU4b2ESGEEGJNPOyPSGykVLGPEuoZ5JTy8nK4XC40a9YMALB48WI0a9bMZ+gAwKhRo5CRkYElS5YEPc6MGTNQUlLi+9DQIYQQQkDPoBSF9hEhhBASY9LQPkoZMaimpga33XYbfvOb3/jUtbKyMrRu3dovXVZWFkpLS1FWVhb0WHfccQfKy8t9n23btsU074QQQkhKQDEo5aB9RAghhMSBNLSPUmI1sfr6elx44YXQNA1PPvlko4/ndrvhdrujkDNCCCEkjeBqYikF7SNCCCEkTqThamJJLwaJobNlyxYsWLDAb85d27ZtsXv3br/0R44cwf79+9G2bdt4Z5UQQghJbSgGpQy0jwghhJA4koZiUFJPExNDZ/369fj444/RokULv98HDRqEgwcP4ttvv/VtW7BgATweD0455ZR4Z5cQQghJbbQGZy7QWkOic9wkoX1ECCGExBknNlKK2EcJ9QyqrKzEhg0bfN83bdqElStXorS0FO3atcP//u//Yvny5Xj33XfR0NDgm+deWlqKnJwcHHvssTjrrLNw2WWXYdasWaivr8c111yDiRMnBl0pgxBCCCFBoGdQUkD7iBBCCEky0tAzKKFi0LJlyzBy5Ejf95tuugkAMHnyZNx99914++23AQD9+/f322/hwoUYMWIEAODf//43rrnmGpxxxhnIyMjABRdcgL/97W9xyT8hhBBCSLShfUQIIYSQWJNQMWjEiBHQNC3o76F+E0pLSzF37txoZosQQghpmtAzKCmgfUQIIYQkGfQMIoQQQkjaQjGIEEIIISQQikGEEEIISVs8DQ7FoNQIkEgIIYQQ0iic2EgpYh9RDCKEEEKIDj2DCCGEEEICoWcQIYQQQtIWikGEEEIIIYFQDCKEEEJI2kIxiBBCCCEkEIpBhBBCCElbKAYRQgghhARCMYgQQgghaYvmMIC0lhoBEgkhhBBCGoUTGylF7KOMRGeAEEIIIUmCjHo5+djkySefxHHHHYfi4mIUFxdj0KBBeP/994OmnzNnDlwul98nNzc3GqUkhBBCCHFGjOwjIHE2Ej2DCCGEEKITw2liHTt2xJ///Gf06NEDmqbhueeew7nnnosVK1agT58+lvsUFxdj3bp1vu8ul8t+3gghhBBCokUMp4klykaiGEQIIYQQnQjFoIqKCr/Nbrcbbrfbb9v48eP9vj/wwAN48skn8fXXXwc1dFwuF9q2bWs/P4QQQgghsSACMciOfQQkzkbiNDFCCCGE6HgaHLpB63PiO3XqhJKSEt9nxowZIU/T0NCAl19+GVVVVRg0aFDQdJWVlejSpQs6deqEc889F2vXro1qcQkhhBBCbOHIRorMPgLiayPRM4gQQgghOp4jgCfTWXoA27ZtQ3FxsW+z1agXAKxevRqDBg1CTU0NCgsL8dZbb6F3796WaY855hg8++yzOO6441BeXo6//OUvGDx4MNauXYuOHTvazyMhhBBCSGNxYiM5tI+AxNhIFIMIiTPijudJaC4IIcSCCMUgCXgYjmOOOQYrV65EeXk5Xn/9dUyePBmfffaZpbEzaNAgvxGxwYMH49hjj8VTTz2F++67z34eCSGEEEIaSwRikF37CEiMjUQxiBBCCCE6EYpBdsnJycHRRx8NABg4cCCWLl2Kxx57DE899VTYfbOzs3HCCSdgw4YNjs5JCCGEENJoIhCDnJAIG4kxgwiJMx7QK4gQkqTEcGl5y9N5PKitrbWVtqGhAatXr0a7du0adU5CCCGEEMfE0T4C4mMj0TOIEEIIITpagzMDRmuwnfSOO+7A2Wefjc6dO+PQoUOYO3cuPv30U8yfPx8AMGnSJHTo0MEXXPHee+/FqaeeiqOPPhoHDx7Eww8/jC1btuD//b//56hIhBBCCCGNxomN5MA+AhJnI1EMIoQQQoiO5wjgceA07EA42r17NyZNmoSdO3eipKQExx13HObPn48zzzwTALB161ZkZBjnPnDgAC677DKUlZWhefPmGDhwIL766qugwRQJIYQQQmKGExvJoWdQomwkl6ZpmqM90pCKigqUlJQkOhuEEEKIJeXl5bYDEEaCvAfLX5mC4vwc+/tV16Hk17Njnj+SGGgfEUIISWbiYX9EYiOlin1Ez6A44ALQ5BU3QgghyU8MPYMIIYQQQlKWGHoGJQqKQYQQQgjRoRhECCGEEBIIxSASCfQKIoQQkhJ4HAaQ9jgLkEgIIYQQkpI4sZFSxD6iGEQIIYQQHc8RwONylp4QQgghJN1xYiOliH1EMYgQQgghOhSDCCGEEEICoRhECCGEkFCk9KIBFIMIIYQQQgKhGEQIIYSQUKSsEARQDCKEEEIIsYJiECGEEELSFq3BmRikpUaAREIIIYSQRuHERkoR+4hiECGEEEJ0PEcAj8P0hBBCCCHpjhMbKUXsI4pBhBBCCNGhGEQIIYQQEgjFIEIIIYSkLRSDCCGEEEICoRhECCH2SelVlQhpilAMIoQQQggJhGIQIYQQQtIWT4NDMSg1AiQSQgghhDQKJzZSithHGYk8+aJFizB+/Hi0b98eLpcL8+bN8/td0zTcddddaNeuHfLy8jBq1CisX7/eL83+/ftx0UUXobi4GM2aNcPUqVNRWVkZx1IQQoJBryBCUgzPEecfEnVoHxFCCCFJRhraRwkVg6qqqnD88cfj8ccft/z9oYcewt/+9jfMmjULS5YsQUFBAcaMGYOamhpfmosuughr167FRx99hHfffReLFi3C5ZdfHq8iEEIIIekDxaCkgPYRIYQQkmSkoX3k0jQtKQbvXS4X3nrrLUyYMAGAPurVvn173HzzzbjlllsAAOXl5WjTpg3mzJmDiRMn4ocffkDv3r2xdOlSnHjiiQCADz74AGPHjsX27dvRvn17y3PV1taitrbW972iogKdOnWKbQEJIYSQCCkvL0dxcXHMjl9RUYGSkhKUP3Q0ivMy7e93uAElt26Ief6aMrSPCCGEEGviYX9EYiOlin2UUM+gUGzatAllZWUYNWqUb1tJSQlOOeUULF68GACwePFiNGvWzGfoAMCoUaOQkZGBJUuWBD32jBkzUFJS4vvQ0CGEEEIAaA3ORr201JgTn07QPiKEEEISgBMbKUXso6QVg8rKygAAbdq08dvepk0b329lZWVo3bq13+9ZWVkoLS31pbHijjvuQHl5ue+zbdu2KOeeEEIISUE4TSzpoX1ECCGEJIA0tI+a5Gpibrcbbrc70dkghBBCCEkaaB8RQgghTYek9Qxq27YtAGDXrl1+23ft2uX7rW3btti9e7ff70eOHMH+/ft9aQghhBBiE3oGJT20jwghhJAEkIb2UdKKQd26dUPbtm3xySef+LZVVFRgyZIlGDRoEABg0KBBOHjwIL799ltfmgULFsDj8eCUU06Je54JIYSQlIZiUNJD+4gQQghJAGloHyV0mlhlZSU2bNjg+75p0yasXLkSpaWl6Ny5M2644Qbcf//96NGjB7p164Zp06ahffv2vhU1jj32WJx11lm47LLLMGvWLNTX1+Oaa67BxIkTg66UQQghhJAgeI4AHgfjRB5P7PLShKF9RAghhCQZTmykFLGPEuoZtGzZMpxwwgk44YQTAAA33XQTTjjhBNx1110AgFtvvRXXXnstLr/8cpx00kmorKzEBx98gNzcXN8x/v3vf6NXr14444wzMHbsWAwdOhRPP/10QspDCCGEpDQeh6uJeeyvlvHkk0/iuOOOQ3FxMYqLizFo0CC8//77Ifd57bXX0KtXL+Tm5qJfv37473//29gSpgS0jwghhJAkw5GN5Gw1sUTZSC5N0zTHe6UZFRUVKCkpSXQ2CGmSZABIDe2ckMRRXl6O4uLimB1f3oPl0/JRnOuyv1+NhpL7qm3l75133kFmZiZ69OgBTdPw3HPP4eGHH8aKFSvQp0+fgPRfffUVhg0bhhkzZuCcc87B3Llz8eCDD2L58uXo27ev4zIS59A+IoQQkszE2j4CIrORnNhHQOJsJIpBoLFDSCLJBlCf6EwQkuTETQz6Y45zMeiBuojzV1paiocffhhTp04N+O3Xv/41qqqq8O677/q2nXrqqejfvz9mzZrl+FzEObSPCCGEJDNxFYMc2EiNtY+A+NhISRtAmhDSNKBXECFJRIQBpCsqKvw+tbW1IU/T0NCAl19+GVVVVb6gx2YWL16MUaNG+W0bM2YMFi9eHJ2yEkIIIYTYJQ72ERBfG4liECEkoTibUUsIiSmaxzsn3uZH0+XcTp06oaSkxPeZMWOG5eFXr16NwsJCuN1uXHHFFXjrrbfQu3dvy7RlZWVo06aN37Y2bdqgrKwsumUmhBBCCAmHExvJoX0EJMZGSuhqYoQQQghJIjxw5q7nTbtt2zY/N2i3222Z/JhjjsHKlStRXl6O119/HZMnT8Znn30W1NghhBBCCEkKnNhIDu0jIDE2EsUgQgghhOhEKAbJ6hfhyMnJwdFHHw0AGDhwIJYuXYrHHnsMTz31VEDatm3bYteuXX7bdu3ahbZt2zrIICGEEEJIFIhADLJrHwGJsZE4TYwQQgghOp4IPo05nccTdP78oEGD8Mknn/ht++ijj4LOnyeEEEIIiRlxtI+A+NhI9AwiKYELQJNf9o4QQmJNhJ5Bdrjjjjtw9tlno3Pnzjh06BDmzp2LTz/9FPPnzwcATJo0CR06dPDNp7/++usxfPhwPPLIIxg3bhxefvllLFu2DE8//bSDDBJCCCGERIEIPIPskigbiWIQIYQQQnRiKAbt3r0bkyZNws6dO1FSUoLjjjsO8+fPx5lnngkA2Lp1KzIyDIflwYMHY+7cubjzzjvxhz/8AT169MC8efPQt29fBxkkhBBCCIkCMRSDEmUjuTRNa/IOFxUVFSgpKUl0NgghhBBLysvLbc85jwR5D5ZfAxQHj20YuF8tUPKP2OePJAbaR4QQQpKZeNgfkdhIqWIf0TOIEEIIITox9AwihBBCCElZYugZlCgoBhFCCCFEh2IQIYQQQkggFIMIIYQQkrZocGbANPmJ5oQQQghpEjixkVLEPqIYRAiJGTkA6hKdCZJUZCBlBkuaJvQMIoQQQggJJA09gzLCJyGEkMig2kzMZABwJToThBBCCCGENHHYVyMkiXEhZbwMLclOdAZI0pEJfbAkldt1WkPPIEIIIYSQQNLQM4hiECFJTKqLQXzAEDP0DEpyKAYRQgghhARCMYgQ0hicijupLAQBQE2iM0AIcQbFIEIIIYSQQCgGEUIaQ1MTg6oSnQGSdGhI/Xad1lAMIoQQQggJhGIQIYQQEjkNoBiU1FAMIoQQQggJhGIQIaQxsBNMmjr1ic5AHEjpWF8UgwghhBBCAqEYREj8iXXHKqU7bklOJnRPEEKIMxL2XKIYRAghhBASCMUgQuJPBmIrKMT6+CpNTXRyA6hOdCYIiTONvc8zoL+c66KQF8docGbANLWHGiGEEEKaJk5spBSxjygGkSZPityrKQmXECdmMpAygyUJJWH3Dj2DCCGEEEICoWcQIfEnFl47aoc0Re7VlIR1S8y4oXu8cPpgcFzQn1EJgWIQIYQQQkggFIMIIcQ+9LoiZugtluRQDCKEEEIICYRiECHpQYrcn5ZkI3VWZKpNdAZISpEuwdwbWw6nYXuiiebRP07SE0IIIYSkO05spFSxjygGEZJipJJnRTp07El0aQptIhqiVqLqyePRP07SE0IIIYSkO05spFSxjygGERJnJBZIpM+IVIq1wmDBxEwtgreJdBGKGtvmPQCORCMjEUDPIEIIIYSQQOgZRAhpNI317EmlDnMqeTGR+JAi78aEQ88gQgghhJDkgZ5BhJBGkwFdJInUwydFni0AgHwAhxKdCUJSkESJQfQMIoQQQggJhJ5BhJBG44J9j5lUD6ibnegMEEIcQc8gQgghhJBA0tEzKCN8ksTR0NCAadOmoVu3bsjLy0P37t1x3333QdOM7rGmabjrrrvQrl075OXlYdSoUVi/fn0Cc02iTbpNNXIqBsWDWD0IDsfouCR1SeqXDiEpBG0kQgghhDSGpLbLH3zwQTz55JP4xz/+gR9++AEPPvggHnroIfz973/3pXnooYfwt7/9DbNmzcKSJUtQUFCAMWPGoKamJoE5J9GkKYtB8SJWDwLehcSMTJMkyYlHM0a+bH0cuC7OmDEDJ510EoqKitC6dWtMmDAB69atC7nPnDlz4HK5/D65ubmNLGV6QBuJEEIIiR+ObCSHUzsSZSMl9TSxr776Cueeey7GjRsHAOjatSteeuklfPPNNwD0Ea+ZM2fizjvvxLnnngsAeP7559GmTRvMmzcPEydOTFje05lMxHdFq3TrODpZJSheU8Ri5cloJ/+pPhWOOIPXOrmJZcygzz77DFdffTVOOukkHDlyBH/4wx8wevRofP/99ygoKAi6X3FxsZ9B5HKl21shMmgjEUIIIfEjljGDEmUjJbUYNHjwYDz99NP46aef0LNnT6xatQpffPEFHn30UQDApk2bUFZWhlGjRvn2KSkpwSmnnILFixcHNXRqa2tRW1vr+15RURHbgqQZ8RaD0o2mJAbZgWJQ00IDr3cyE0sx6IMPPvD7PmfOHLRu3Rrffvsthg0bFnQ/l8uFtm3b2j9REyEWNhLtI0IIIcSaWIpBibKRkloMuv3221FRUYFevXohMzMTDQ0NeOCBB3DRRRcBAMrKygAAbdq08duvTZs2vt+smDFjBu65557YZTzNibdwkOk9JzuQhKQ+vI+Tm0gDSJtFA7fbDbfbHXLf8vJyAEBpaWnIdJWVlejSpQs8Hg8GDBiAP/3pT+jTp4/9TKYpsbCRaB8RQggh1kQSQDoS+wiIn42U1DGDXn31Vfz73//G3LlzsXz5cjz33HP4y1/+gueee65Rx73jjjtQXl7u+2zbti1KOW4aJEIM4qQAQtIDikHJjYx6OfkAQKdOnVBSUuL7zJgxI+R5PB4PbrjhBgwZMgR9+/YNmu6YY47Bs88+i//85z948cUX4fF4MHjwYGzfvj2axU5JYmEj0T4ihBBCrImHfQTE10aKyDPo559/xlFHHRXJro74v//7P9x+++0+V+Z+/fphy5YtmDFjBiZPnuxzidq1axfatWvn22/Xrl3o379/0OPaVeSINfHuzMmUtGwA9XE+dywQYYudYh3WAyHJQ6SeQdu2bUNxcbFve7h37NVXX401a9bgiy++CJlu0KBBGDRokO/74MGDceyxx+Kpp57CfffdZz+jcSSVbSTaR4QQQog1kXgGObWPgPjaSBF5Bh199NEYOXIkXnzxxZiuSFFdXY2MDP8sZmZmwuOt3W7duqFt27b45JNPfL9XVFRgyZIlfhVDoku8O+910AWUvDifN1Zkej9Eh2IQIUmE01Evr7FTXFzs9wll7FxzzTV49913sXDhQnTs2NFR9rKzs3HCCSdgw4YNjShkbKGNRAghhKQhMbaPgPjbSBGJQcuXL8dxxx2Hm266CW3btsXvf/973+oV0WT8+PF44IEH8N5772Hz5s1466238Oijj+K8884DoAdMuuGGG3D//ffj7bffxurVqzFp0iS0b98eEyZMiHp+SOLQkD5Bq11I8vmZhMQQTvlMbhwtK+/Qi0jTNFxzzTV46623sGDBAnTr1s1x/hoaGrB69Wo/T5dkgzYSIYQQkn7Eyj4CEmgjaY2gvr5ee+ONN7Tx48dr2dnZWp8+fbRHHnlE2717d2MO66OiokK7/vrrtc6dO2u5ubnaUUcdpf3xj3/UamtrfWk8Ho82bdo0rU2bNprb7dbOOOMMbd26dY7OU15eLgvc8JPEH5eN31020ybykwtoeUmQD374ScQnB9AykiAfqfYpLy+Pyns13Htw/UBoZafY/6wfaD9/V155pVZSUqJ9+umn2s6dO32f6upqX5rf/e532u233+77fs8992jz58/XNm7cqH377bfaxIkTtdzcXG3t2rUxqYdokg42Eu0jfvjhhx9+kvkTa/tIfRc6sZGc2EealjgbqVFikFBTU6M9+uijmtvt1lwul+Z2u7Xf/e532i+//BKNw8ccGjvp8UkVMcgNXRBKdD744ScRn1xQDIrkEy8x6KcB0HaeZP/z0wD7+QtWttmzZ/vSDB8+XJs8ebLv+w033KB17txZy8nJ0dq0aaONHTtWW758eQxqIHakso1E+4gffvjhh59k/sRTDHJiIzmxjzQtcTaSy3vyiFi2bBmeffZZvPzyyygoKMDkyZMxdepUbN++Hffccw8qKipi4hodbSoqKlBSUpLobJAY4YJ+NyULatT2IxEeIwPxX9XNKU7qPdmuEYkdxQCqEXnbb6qUl5f7BSCMNvIe/LE/UOQgqNmhBqDXytjnLxVJBxuJ9hEhhJBkJh72RyQ2UqrYRxGtJvboo49i9uzZWLduHcaOHYvnn38eY8eO9QUy7NatG+bMmYOuXbtGM6+kiZJuQoEHepnsxE6JtOzJUGdSxmQXrUh8yQDjBiUzmgfQHFwgjTd4ALSRCCGEkPTDiY2UKvZRRGLQk08+iUsvvRSXXHJJ0ABFrVu3xr/+9a9GZY4QAHADaMx6LIkWRczIs8HOsyRY2cM9X/K9+2UAqLeftUZhFqA8AEoB7Lexr5NrlAxCF4mcZLp2bEuBUAxqPLSRCCGEkPSDYpCX9evXh02Tk5ODyZMnR3J4QvxI12XY7XRCIy27eF/E0wPDqmOdHcfzk9RAJkEnAxSDAvF4AI+DB4fT1TKaArSRCCGEkPTDiY2UKvZRRCtcz549G6+99lrA9tdeew3PPfdcozNFwtOUplq4E52BGGHn5ot0CfrDABoQP68gwFq4qo5wv1Cw857a1CN5rmEyv6cT9XzXPM4/xB/aSIQQQkj6kY72UUR9zRkzZqBly5YB21u3bo0//elPjc4UCU+8vT4SSaSCSDoQ6TU+gvh7YFjl1Y4Y1ZSvb1OkAckjBiUziXq+ezzOP8Qf2kiEEEJI+pGO9lFE08S2bt2Kbt26BWzv0qULtm7d2uhMNWWyYG+VHQ8S26GSjko88lBnM1029HppcHj8XO85pCzxKJM6PSUD0RVusqHXQbyfQVbtNpwY5IJe//H0YCLRxelUKxEq05lMOH8OqSRS7Oc0scZDG4kQQghJPzhNzEvr1q3x3XffBWxftWoVWrRo0ehMNWXMFyRYe4tXZypUpyRenRW7QkEmImvQOUhMfB0h1JS/SPKUGeF+jcXqmReuQ+wC4wqlOk7bWqKF7HgQDW83ThNLXWgjEUIIIelHOtpHEdmsv/nNb3Dddddh4cKFaGhoQENDAxYsWIDrr78eEydOjHYemxTmznMknaYM7ycagZdDeazEq0Nnx1MKMPJqtxMl9SN1Hs9OquQ1M8x5nayilqn8TZWpVxr0+EZA+gYKTwUa015icc9kI7XbQ2PrRENyxzMioaGNRAghhJBUIKJpYvfddx82b96MM844A1lZ+iE8Hg8mTZrE+fCNpDFTC4RM6J2JjCgdL9Ecgb2pKNJ5slNul5LObgyTaK48JMfJROhpcLUOjilTUxLlGRQJGgzBq7FTa0jkNKa9xEIMykJkUz6ThWjUScLEIM3haFa6u3lFAG0kQgghJA1xYiOliH0UkRiUk5ODV155Bffddx9WrVqFvLw89OvXD126dIl2/kgQRPBR26OIFdKBimUbVI8d6068k3KEG1HPhj7tLEf5324Mk1jUZ6hOuMTTOQz/GE1Z0Ou8FkCe9/c86OURYSmVvAqkXlO1458OJGN7SZF3qCXJWJ928Xic5T9V5sTHE9pIhBBCSPrhxEZKFfsoIjFI6NmzJ3r27BmtvBAHyBQK1atExKB4tz0JWJxo7AS1lmDFudAFoUrYn4YWC1zKX3O+MwHkQxd7ZBpPA/R850AXgwqhe9YUQL8OFd50KfL88SMZ2lC0iKYXWTxItrzGeyW8SAl2naOR90S1Ic3jUIBPxYdNnKCNRAghhKQPTmykVLGPIhKDGhoaMGfOHHzyySfYvXs3PCbpa8GCBVHJHAmOVZyZWHmuhDturDvxdjtFwdJkwRDJ3AAOwfAKigcSw8lKdApVdx4YYpEEWW6ALmTJcUsA7IPhIQTo5UqWjnQBgKpEZ6KJkGoCVCgSsRpespGoa0nPoMZDG4kQQghJP+gZ5OX666/HnDlzMG7cOPTt2xcuV6pEKEkfrISFaHceQokYKrEWVezGPlK9CTJg3Kw53r/iSbMPusfN7ijmMRSZ3jyEWnrd6tqpz5A8GPF1Crx/swC0BfAzgGYADnjTHkZw1HqJBy3QNMWgSAO/N+bapEuMMCB+Qm1jSRfxTYWeQY2HNhIhhBCSftAzyMvLL7+MV199FWPHjo12fggJwA2g2kY6VQwy36hyP0pA5nrEr+NsNXXPrhfHEdP/GdA9gw55/8o0QbvliXfnNdlXhEomb5porECVTOVpyqTydaBnUOOhjUQIIYSkH/QM8pKTk4Ojjz462nkhSYYLzlcZshO3xynNYU8MUsUQOb8aVFsDsN/79xDi533gsTiXBN0OV09Sbg26h00OdC+gjQA6Ayj3Hr/C4hxWNPa6OO3k5tjYJ5Ed52TypmlsHXiglydVRYhUR332ZSKxscgaAz2DGg9tJEIIIST9SEfPoIzwSQK5+eab8dhjj0HT2O1IZyIJ4hqLFtFY7xK1HA2mv/HAatU3u8h+IhzJVDFZdluWZa9F4kSAUOWxim1lJpFPkYgegCRqBGs7qTipJl3ehh6P8w/xhzYSIYQQkn6ko30UkWfQF198gYULF+L9999Hnz59kJ2d7ff7m2++GZXMkfiixiyR/yNpx9E2f3OU/yPxIjkCY5ReylMTJG0sMItqGcr2cEi+q5TvP3v/36/8fgDxiQVkLocsdR/MK8mOR1ciyUTqxKexQ4q8d3xkw39FRCELhuAZa6IZRysduv70DGo8tJEIIYSQ9CMdPYMiEoOaNWuG8847L9p5IQ6IxdQatVOUTCPzsqpWpOWN1dLPkZIB5/Ur10WDEffISuSKJ8mw0lxjSYfOOyHRxKM5G83y8CYKgDYSIYQQkn44sZFSxT6KSAyaPXt2tPORlMR75SU7SAc8G3pHO1qdbRf0VarKvd8lpk0ysB+GB0Ek09aSpRyCxNGx8ogIhwZDAIqkPqJJFnRhKlRslEjKGA+kfSdr/lKdUM+PTBjTB4N5ZcUz3o6dZ7xTMTqS94YLyRHDSvMAmgO1mjOhAmkqNhIhhBDSlHBiI6WKfRRxyIwjR47g448/xlNPPYVDhw4BAH755RdUVlZGLXMkEGl/mYh+vBP1eMnkGVSP9IrtEklgbiuS5RkTKh/JJqYKcg2SpQ6bKsHqP5J4ZbEiWvdrqsCYQdGBNhIhhBCSXqSjfRSRZ9CWLVtw1llnYevWraitrcWZZ56JoqIiPPjgg6itrcWsWbOinc+EkKhrmI3gI+Y50OPdZCP0yLpTxONEOsjxiKMSqpwqNQCKEN84P7Egkml4uYhPuVVvDbvY8ao5HFl2okIosSfSzn0ivQWTWbwy10so75ZEe744RYNz8TDSWGvq8yFULK6Y4tAzKGkbZQJpKjYSIYQQ0qRwYiOliH0UkcPF9ddfjxNPPBEHDhxAXl6eb/t5552HTz75JGqZa6qEuigZyl+7bTGSNhtph9dJH8JuWg90saIpjc4LjV1JzS6ReD/YecYlUhSPRXtpim3QDunkuZdI5J6SKWOJgJ5BjYc2EiGEEJJ+pKN9FJFn0Oeff46vvvoKOTk5ftu7du2KHTt2RCVjTZlQAoAED66C/yhyqI55qN/UEf1otNlgqwNZYTcuSAGAQ0gZgTUo6kpmJTb3yYGxklgskaXrnZCF8NewGMC+iHJkEKk3Tqh9jsBeec1lTORzPRHtX8SvcOd2IlwUQfcYU+vV7nkSRSKue6LqwuMBPA5Uz1QJkBhPaCMRQggh6YcTGylV7KOIBh89Hg8aGgKd/bdv346ioqJGZ6qpE6qNSa3XK/83xltB3TcabTZcg1LPZ7eDlYP0CvR7BPZvvIjU2giI5NrbKYM7guMC/u0kFt44dstr9sBLked61LDrMebkGrlh3XbodZUcaB7nH7vMmDEDJ510EoqKitC6dWtMmDAB69atC7vfa6+9hl69eiE3Nxf9+vXDf//730aUMPbQRiKEEELSj1jZR0DibKSIxKDRo0dj5syZvu8ulwuVlZWYPn06xo4dG8khiUJt+CR+WHVQnUzBEtTVqSKdnhQuxkUknelIO4k5SN7pK3afD07bQjyx49lVrvzvZOpLNKYsRgOzB1G6ChbBREe7gZydrP6Vj8DnSzwDRocSokJd33he+0SKjrEUgz777DNcffXV+Prrr/HRRx+hvr4eo0ePRlVVcP/Hr776Cr/5zW8wdepUrFixAhMmTMCECROwZs2aKJQ2NtBGIoQQQtKPWIpBibKRXJrmfOGz7du3Y8yYMdA0DevXr8eJJ56I9evXo2XLlli0aBFat27t9JAJpaKiAiUldifupAaNDXQbb2+cUNNEWgLYG8ExC6BPyUqGgLXm8pUC2G9jvzwkNghzNBEvm2S4HpGSzEGcG4Mb8RMeewDYCSBRaypZLXsvz8sMBBem4nntrQJIl5eXo7i4OGbnlPfgewAKHChfVRowDpHlb8+ePWjdujU+++wzDBs2zDLNr3/9a1RVVeHdd9/1bTv11FPRv3//pA3EnE42UjraR4QQQtKHWNtHQGQ2UmPsIyB+NlJEs1A6duyIVatW4eWXX8Z3332HyspKTJ06FRdddJFfsEQSO6w6LVne75HEfzETqsOudhyj1UEKdQy7olQO9HxL3qNRD9HCnI8G+At2Vh1Uq/2SDTXfVm2hCHq8JyD63h/xWtlLLWOyX49IiYXXi1VMqSyb51KfZdHEBes2oyl/Qy13Hy/UlcVShYqKCr/vbrcbbnfoiaLl5brvYGlpadA0ixcvxk033eS3bcyYMZg3b15kGY0DtJEIIYQQAkRmHwHxs5EiDkmSlZWFiy++ONLdSRjCiSzSEVZXn8nyboulGOSC7q0SbTEoFHaXV86BfyylZBKDzMgKaaoYZLW0e7zz7zSQbxaM+s5AYLsxi0HRJB5ikNxX8fBmCnUvhbvPGnsfxkoMMt+D2Rbnssp7NvyF3WhhRwxKFhLlQacBcOIvLEk7derkt3369Om4++67g+7n8Xhwww03YMiQIejbt2/QdGVlZWjTpo3ftjZt2qCsrMx+JhMAbSRCCCEkvXBiI0VqHwHxtZEiEoOef/75kL9PmjQpksMSBTur91iliUcHQvXUiUfnSYP9zq7a0ZSOaDCvm3iQC32qmhm79RavpeUFqetgmAUYtV6tOtl2xJpIr0+8PCfiGcsm0nba2Dw6ifljF6vrcwS6YKvm16rcdld7c0o8p3k1Jj5aIkUpD5zdW5J227Ztfm7Q4Ua9rr76aqxZswZffPGF4zwmO7SRCCGEkPTDiY0UqX0ExNdGikgMuv766/2+19fXo7q6Gjk5OcjPz6ehEwcyYT2NItaihwb/GDbJ1lFWg8PKTSheBolA4haZMXsBBavHeK0mphLqmqreTIC/iBCJOOlC5NfH7sNY2oTd9GpHPhZTlYKdM1Qbd3KfRSJE2PW+c4KVZ149dK9C9VpYiYGxyA8s8hOOxog6jVmhL5FxtSIVg4qLi23Pib/mmmvw7rvvYtGiRejYsWPItG3btsWuXbv8tu3atQtt27Z1kMv4QhuJEEIIST8iEYOc2EdA/G2kiBZbOnDggN+nsrIS69atw9ChQ/HSSy9Fcsig7NixAxdffDFatGiBvLw89OvXD8uWLfP9rmka7rrrLrRr1w55eXkYNWoU1q9fH9U8xAOn0zQaENgYYyHM2M1XrFftEg+fUIgwYc5LPASrYOWvt/jNKt6T1RQx2e4EERVykRwrX4ULShwPsaWxsYpCeUuZt5uXondyDnMe7R5HXf49Ga65EKzO6+HfrhMVGydZ6krNRzxXVQuGFsHH9rE1Dddccw3eeustLFiwAN26dQu7z6BBg/DJJ5/4bfvoo48waNAgB2eOL7SRCCGEkPQjVvYRkDgbKWp9+B49euDPf/5zwIhYYzhw4ACGDBmC7OxsvP/++/j+++/xyCOPoHnz5r40Dz30EP72t79h1qxZWLJkCQoKCjBmzBjU1Fj5YyQvTi9EHQI70bHoRFgJMFbniaUHi4gnOWHSyRU312U8OpvZQbZXIzA/amwnIZgg4nT6Tob3+M1g/5o4aXvqtbfTmT4UZLu6b6y8QATHHVbTd4nvZEaEN5Usi21q+lCo11quox0yYeQv2PTRRBDsvquGf3uPxRQ1O9hp95HWpZNnjtmbMdGBo2MpBl199dV48cUXMXfuXBQVFaGsrAxlZWU4fNjwN500aRLuuOMO3/frr78eH3zwAR555BH8+OOPuPvuu7Fs2TJcc801jSxpfKGNRAghhKQ2sRSDEmYjaVFkxYoVWlFRUdSOd9ttt2lDhw4N+rvH49Hatm2rPfzww75tBw8e1Nxut/bSSy/ZPk95eXkk9m9UP5lROEZWAvPljmHdFAKay+Y5CgAtuxF16wK0DKUuM2zulxNke4bp/C5v/sx5dHrcUNcrB9B6AFqLIOdxWXw3b7PTHuzuE8knlseO5GPVhlwW27MQvM04KZML9u/nLDhvr4n85Aepz3h/kqWuQuVDbTPl5eWNep/afQ++DmjvO/i87iB/wco5e/ZsX5rhw4drkydP9tvv1Vdf1Xr27Knl5ORoffr00d57770olz4+pKKNlAz2ET/88MMPP/wE+8TaPlLfhU5sJCf2kaYlzkZyeU/uiLffftvvu6Zp2LlzJ/7xj3+gU6dOeP/9950e0pLevXtjzJgx2L59Oz777DN06NABV111FS677DIAwM8//4zu3btjxYoV6N+/v2+/4cOHo3///njssccsj1tbW4vaWmMCS0VFRUCk73hjtQxzKpEPfcQ/FjQHcMBm2kLoXlN2lqOX6TXqSHwWdC+LAgD7oZfrMPS70Wp/2W43CLLVCkvRJgfASOheOT9Brzs1b8GW/A7nlRCPleOEVLgfwsX5iRfiFZTofNglF/7eYFb5bmwg5WQIxBwKu/eSeh+Ul5c7mnPulIqKCpSUlOA16M89u1QD+BVin79UIpVtpGS0jwghhJBgxMP+iMRGShX7KKLZPRMmTPD77nK50KpVK5x++ul45JFHopEvALoh8+STT+Kmm27CH/7wByxduhTXXXcdcnJyMHnyZN+yaU6XVJsxYwbuueeeqOUzGsSy0yKih0iMZuEiGp38WOZf4rDYOUdjplhkQb/BPXDemXRSfidpI7k2suKXiE7BpiepK4PFuv2FOn48loiPFfEUG4K1yWQVPIDg1z5cm5BA5aH2DXWMZBaDnMQqSkT+5T3hJD3xJ5VtpGS0jwghhJBkwImNlCr2UURikMcTn66bx+PBiSeeiD/96U8AgBNOOAFr1qzBrFmzMHny5IiPe8cdd+Cmm27yfU+GkS+no/qympidKyHBhBuge8wUAihXfo+GGBSL6APieZEJwA19hFzqKVh+DyN4/B4zZqGkFEAHALsBVHm31YQ4l+oVYveOUINc23mgZMK5h4wbwDYA+6CXw9z5lOPlQQ/wrNZrKCJtI7nwX4FOxQVdgKs0bY+lV1C49m73fnDaaQ6FHU8oN/TrZI6xFM6bC3Cez2h5gWUhML9Z3uOHembkQq8PNY2IhvI3G8E9AO2sIBdtT7dYiJqJ8PaiGNR4UtlGSkb7iBBCCEkG0lEMivUiUI2iXbt26N27t9+2Y489Flu3bgUA37JpTpdUc7vdvmXenC73lmjEy8fuqkUu0wdwNjJtl1g1eHOHTcoRLFCvk5tU6iHP+38uDCEpnOjUWJxcP6dkQ+9EH4beoQ7mXRGP1ZTidZ5okqz5jSRfyV6WYPeXGhTbvE+w75HmgfjjieBDEkMsbKRUto8IIYSQWJKO9lFEnkHqqFE4Hn300UhOAQAYMmQI1q1b57ftp59+QpcuXQAA3bp1Q9u2bfHJJ5/45sNXVFRgyZIluPLKKyM+bzDiMfUglIeAeMhkeNOEilPjgn+HSl3i2zwi76SxOonlkgN95F5GzO3G1VFxQfcaqYNR7xkAOkKPhXMYuueB5j1fAwJXZbIqXyZ0EagKwCAAiwG0837fYzOfjRm1L4R+HerDHMeqroN5NEh9N4Pu+XUYwcuej/DLvkcDN8LfL6rXkLQRs+dUNvS6CjV9yC6NifMTSRu2g932pkHPvzwDYoXUb2OfeVZ5lGdCqGO6LbZJvC073njh6lMEZTuxxaz2tcq73edouCmTTo4VC+gZ1Hiaqo1ESCoRzziIhJD0IB09gyISg1asWIEVK1agvr4exxxzDADdAMnMzMSAAQN86Vyuxo293njjjRg8eDD+9Kc/4cILL8Q333yDp59+Gk8//bTv+DfccAPuv/9+9OjRA926dcO0adPQvn37gDn7qUKoGlOnTTXAPxaQmQzlo8G/g92YzqzdK6ouuy37RNoJPwKjcyTHEi+eWhhTqSQIstmTyJwviZuUAz24Vytv3nKgC0yRdBCdIp5NkdwhInBp8Be7RDDJhn+nGQjs1KvHiCUi3gTD3B7N3mtqgO5kCCgdK28Sp9PSUsWrJVLRxMqjzPy8C1VnduozVeow3jgdzUqVka94QhuJkOSGz39CSCQ4sZFSxT6KSAwaP348ioqK8Nxzz6F58+YAgAMHDmDKlCk47bTTcPPNN0clcyeddBLeeust3HHHHbj33nvRrVs3zJw5ExdddJEvza233oqqqipcfvnlOHjwIIYOHYoPPvgAubm5UcmDinQworXSkZWQYz6umqYBhldPFvTR83oExuTIgB7/RrxmZKqQdLojzbt43thFzXckaNDzmgcjvk899JurEnr56wC0BLAL+qpjv3j3lRe9+dwlAA5Cjw1U7D1+e+9vVcr+oRAPnMaQB/1ahqqbfOjXshJGnBURe7K829sCKIMujnUF8L33d6vVwsSbxAO9rNHwcBEBKhgl0GMXhUPaubSxHO/3Gu858mBc+8YKWOEezo3xOIklck2djEo09kUUKkCzKkyZV+ULRYOSNpinVx0C27C53I25FhpCt9tw+wpSD1kIFDatvBLDxRWS4PWx8kCzAz2DGk9TtZEISRX43CKEREI6egZFtLR8hw4d8OGHH6JPnz5+29esWYPRo0fjl1/sdKmTB1kuzi7hOsB2sDv9Ipjnjxv68udVCJzukwFd4DgE3fOlsXkV8qB30ux0UlzQO/Rq3oKJKOFcdYu8x3Epx2sNXQA5BN2zZyv0Mv/s/V3tZKpTUtoC2AngeO9xfwRwOYCZALoBWGujbHkIHhDZDi7ows1eb/6DUQy9k7kf+vWWoLtF3u0HAfQEsN67rTeAJQCOA7AB/kGZ3dDrxKqT3RjcCD3d7CgAO8KkAYw2UAi9zeZ5t1d5z1EEvbx2807379hiFoMi9fqTqV/ma1UM/bml3mfREGGjjSpieuDfPq0EHbtBpq0GHOK1tPxzcL60/GQk/9Kp8SSdbCSn9hEhhBAST+K5tLwTGylV7KOIAkhXVFRgz549Adv37NmDQ4dCdW9TE3Mw02iM2IZSFrNN6ayoh95RDib01EFvhNEcXa6DvY6MiC/mvB2BtWtuuE57C+idQ3XfQ9Dj4oiXi3nFqvYwpsl19e7fCkAf6OJCb+jlcUEXgOqgryJmh8Z2SO1M1ROvr1LoeSzyfjKgCyz5MMqTAV3IOgC9PvYjMC5UPewLeWYylL/qA8OO4JJrkSYTwYMB10AvQx0MAane+78TcSfHQdrGEmt3c4lvE43jhMLqugRDM/1v1yvIfHwRgszBomsQeJ9ZeU2a26MTnLz8zG1fUKesqdM1Aes6sduGE+mBpkXwIf40NRuJpDbmabnybE3qFWYIISQBpKN9FNGz/rzzzsOUKVPw5ptvYvv27di+fTveeOMNTJ06Feeff36085hwzC/FWM8BNHeMrPBA7yCbY+kIMn0sGnlVp1w5icVhPnekeSmELiqoZTzs/WjQ68G8zHQJjJg8Lb37F0OfHpYFPVi0dC53eP+vsJmfaImBoTqv4hlQAGOlMwke3hp6eYugl8cFXRiq8u5bicCOs3gtRPJgUlewc9r5zrbYZtW+5VhHlL/yfyR5t3MPRYNYrtCnnqOx5TELJ1Y4EYPMmK9NuNhn5v3MebMKrG415cq8n5P8O0kbLr6X+tLPVLZZpbNDIg0IribWeJqajURSG/PzzWWxjRBCSHraRxENOM+aNQu33HILfvvb36K+Xvf/yMrKwtSpU/Hwww9HNYPJQDwC7arkItCzw4ys0iQCiJo/Dxo3jclMqLJLR1UVH4I1/kin7mTAX1SQKWAybUo8kMzeCoDuVdQVwBboXjb7vPtugx5rpwrAdm/aSKZPmctkFUtF/pfOrQQBDzV9Tx4iR6CXXaZ8ZUIXgQ5BX1Ftj/d8+6FPowJ0j7DGok5nkfafCSM2Sob3f6spiubrb+7Uq/Ul04TCPTBlxbhQqHUframRoZC6kOl3jSXY/SGeQVLXTlcTk2tirmPz+SIVC60IdZw86NdcbTuh2kioc1jd86FQyyxtz7zdKm2+9/9gPh0StF29zxtDlvecdgXqaMKYQY2nqdlIJHWxClngsdhGCCEkPWMGRSQG5efn44knnsDDDz+MjRs3AgC6d++OgoKCqGYuWYinGOSC9bLKZiRGRbBOaDgxKVpIR1XtmNoJOhvJOdRVyeQ44WIQNYPuPaN5/z8Ivd52Qo/ZI9PpAHvLUZvzb46VooohqhikdjxFDArVmVfFoCzoopXUQwF0cSgf+tQwDfqUOemoRuPaq+VQPR4kbku293+z6GglBoUSekTYU9NYrZLnNFZQvMSgBkS+RLkgeQ4Wd8fsGWSn/Zj3B4J778j2eIxgaNDFbpn6J0RybqfPEnO7UgW2cPe2PJODiUFyfeyIQXaeg1nQPSITIQZxNbHG09RsJJK6BHs/8L4mhJBA0nE1sUZNCd65cyd27tyJHj16oKCgABHEok4J4lkqDcZ0n3DpzBcvUfO77cZnsepsBYsdoyKdqyzlb6gOlQbdA0iD3onbA13wqYAuAnmgTw1TxZlIcUH3dMiELs7kQfdAagVdfCqGPmVNVV1FVQ63apVMgZMYQw3evP7i/b/CWzYPDK+gxqDWg9U1dcPw4rFawc58DMBaJFHF1WCeME4w1228kBW0GhuQO5wYYz6HB/a9T8IJD+GmK0aTLBhT0SRPTmM7qQJruNEZ8xRF81Q7tb7DTXM7gsC2rR5fvR/EWyxYvdr1YMpQ/iepSVOxkUjqkkpxLQghhESfiPrB+/btwxlnnIGePXti7Nix2LlzJwBg6tSpUVsytSlTGT4JPAjs3MQrTooZuwvUmjueoWJ+qOnMYlA2wndyd0Gvo1zo4kkVdC+ard7jbVKOazdWitX5MqCP4GdBn75VCF0Iagt9ilop9CXvzR3TcAaYdP4Pw1ChZRrKJu/vB6GLWxp0L6fGonpfWXmn5XrP2wD7YpBV4GdVyAjlRWYXqVu7Bm20OtciBkVrhatQ4qZZDIrmqmrRFpHNwUiFbASKQU4Xtw52bCvMQpOcW42BJliJQep56uH/nMiA/z0i7Vw+WWhczA1VDIr3c12EarsfdiQDoY1EUoVwA1OEEEIMnNhIqWIfRdQPuPHGG5GdnY2tW7ciP99YYO3Xv/41Pvjgg6hlLt6IOGEV9DYW57LC7ry9IwjshMb6hW7VEdOgT0tS8223UdkNOivLoaurSwUzYMzBacuhxwaS/z0IrGO7N6wcOxNGR08Vr+q9+ayBPp2kwvv3EIxrleX92J3KVQP/Tms9jKkq1d5PY8QNmYIk07UEK8+Tw/AXIeqg10Oekge1A253GW01L4Dzkcpw5zC3x2g9nCOJsWPu2Ie6dmq+Y7G6lJV3ltMXghPvF3k55sC4d5x6VUU6ii3CnbpvKJFFDZoO6PedGovLyjNTxUpMdRJsXH2+xbujpkXwIf6kq41ECCGENGXS0T6KKGbQhx9+iPnz56Njx45+23v06IEtW7ZEJWOJwA29c5KH2McckTg7ZkNf8iCdkWAdgToEikGhOox2PQTCxXcBAuMDVUCPYyP5VoUFOa/8VfPQAP8OWSaMMqjpZLlxmT5nFRxbjq/GUtFgBIkGdG8hDfp0LrXu7Ha05djZCPTOyPDmMxu66CSxiMzlzvP+rYT1FDnzNa9AYF2IKHZAyVekS8ZLIGgX/OtVDVYs+T9o2r8aenstgR7AWqbMHULgtQCsy6d6i2XBEPqcPEBDpZXyRct7R8V8zGD3rHlaVI3yPVicIEBvS+o0wWDHjBSr6WlW924orGIXBdunwZs2H3obyYTz+FaRllmEJ3X/bPhfCxW5N1zQr3Ot92844Ux+L4R+j6vpzVPcZJvV+VVvsESJQU7SE3/S1UYihBBCmjJObKRUsY8iEoOqqqr8RruE/fv3w+22E/44OZEOl8SHiIYRHuw4wbxR7Hp5yOo1TvOYCf+OiJ0pO3Z+V1flsSpDqCkw4dKEm4ojHVjx2FFRl6iW49vxRjAHglanfKj1J3kzl186ntJZVgUPCcBsJQapnfEMGOKBit2HSziRSAIgy/QzFSeriajChogX4l2nXjertiEPICdt2dyBDrWfXRf4aN3vwe5pVXix0+Ylrd3zqPlXBbxQWAUObYzYooq+VudQp1ip25wKW3bTBxOpZP9Qnl3mulfr1yrouQRWD4daD8GIViyqSGEA6caTrjYSIYQQ0pRhAGkvp512Gp5//nnfd5fLBY/Hg4ceeggjR46MWubijYz8iodHNFDjsKgE64i4gvxvphD2Vh0T5Fxu6GVzw3m8DhEOrJCOkLnDH67TZvYSsUrfgOCr+ADG1KRcBKqbVl5EVttUMuFft+bv0qEXT5YK6CJILfS2Uw3Ds0b2Ew+VfO8+bgTefObzZEH3ujFjt47zgmxXf5f4P+aOrHjGhRIVpIOsxnQq9/5fBMNrTNJaiUG50L3KAPveTdnwrzurTrN4nEmnP1wHPBpxWYLVVai8hnpROBEDJEaN+f9QUyCtYnaFWhUwVB1mmf6q58hS/geMuhYR0ipuWIbpu/mYdqZcyT2oisSZSn6qA/YwkPtC2qT6PpD7SvXgLLLIn7n+5FqYV84zI/dVqPzFEk4TazzpaiMRQgghTZl0tI8i8gx66KGHcMYZZ2DZsmWoq6vDrbfeirVr12L//v348ssvo53HuBLpBRTj37zMuvqbHex66lh5wZhRO00e+I/CW3VCG9Noo9Hg7Y7SB9tP7SCKV04kyzxbXX91m1qXMKU176spf8UTS6ZTHbTIjwbDswMIDLgbKt/mbeGuSWPj0Ej565Vzm6cehVqxSX5XO9pWHjrmbXbampWAEKq8wY5p994Nlc7ONQk3LVRF6sPqejtR99V2ph43HOp5g/1VCdUm5Hih8iXikNnDL9g9GOz8cmwndaR6L5mPpx5XbVvi4Wf3+GreVPEuFnGi7EDPoMaTzjYSIYQQ0lShZ5CXvn374qeffsLQoUNx7rnnoqqqCueffz5WrFiB7t27RzuPcUWW9Lbj8q9Wnht6B6AYgR0HJ5UsXitWgo35fOGUvBzoXhcF3v8lcKvEG3IaryNYXgD/DlKkwlCw/Y7A39tBRQI3S6dN9a4pRPhOmVUdmqelyVLqIvLJlCarjqXamXPB8E6QwNXl0MWPNhZ507zHzoV+rWS6YjDvAdX7w6ostQhNVZjfw5EFPc+y+p3UWab32GpbEAHB3Gmugf8qTXkIrFPzqlD1CB883MrrCiF+D+aFYzWdzwp3iHRqmYO9GKw89YJ5Dlotxy6ehjmwv/S8PA+EXIR+VqkCpaBOrVX/qvuYf5PnjogfVku2m+MYBfPMcxqMORuB4mOoY4iorOZHPEjVTwUM0SoThvdQuPyp11gCscv1i0WcKzvQM6jxpLONRAghhDRV0tE+cuwZVF9fj7POOguzZs3CH//4x1jkKWUwx+lQ/wpOxSBJH85zJ1QDk5F1WbnKytvIPPJu9T2Yt0OwuEB28hYJ0mEM17FSY5HId7sxmMxYeQaZv9spp9mjxRw0O9Qx5RpYEewamOPTWG1Xjx+OUJ4XqreT+rsaVFim0gU7hni4WB031g9RO+cI93u4Y0S7HFZeNKoAqda3VT7iUa8qVueqQ6DYEyp/oZ43djzmzPdUJOVX91HjaamelqpnYrB7PFw8JzX/8b5WAj2DGgdtJEIIISQ9oWcQgOzsbHz33XexyEtK4YI+wixGv3hJmA39I7A/ZUCOWRDkd/W4h2Hd6ZO4NG4YXkFHYCzNLp5HaudKjaMB6B4DWQgeU0jyaYUa+NRp4wq27HYtgk+Lk5F7FwzvLBlRN3vpqOeR72ZPBjXWjJoPqw6mrHJmhdoe5Bg1MLyYDljsI94ANfAPQG32ngjlQSWeCTDlLRvW3lXhvkuduhCoHNcrv6nnKoLxsGyO4FO0VE8T+V9Wd8pTyqF6OKkxaMyonXDzvWH+Hk6kFSFRje1lFQ9H7gPVOy2Uh5Jss/JMqkGg6GG+Zhkw2rcG/R5Vp6gWwnoalngTqd5AtfD3Pqu12NcKu7GMrO6bTAB74e+VWAKjjHnw94iR+rU6p6zGaPauEY8hKwGpHv7tKVRMJfU86vmlLUsgeEE8g6S+rUSfAgT3aKpXjpeJ4O+BWEPPoMZBG4kQQghJT9LRPopomtjFF1+Mf/3rX9HOS8phng4mI8Sh0oU7nt14E+ZG5oL//up3D6zz5wrzfyjvk3B5s5PO/HuwjrQ5Rk+o40k5g3lXqfWi5jVYniRNMM+YYPmwQn04WAl55jRA8Bs0lKeO1flVMS3Ydbci3APCHCvIvE+4qTzmcqgruJnTSNmC5Ultt3Y8u0Lly6p92CmHVZsOdXxBxDM77VFt11ZinlXbsLr25vNFewQj2HUyT/NTxVkRUc35tXv/2fFadLJqXahzhRMUgx0zWB5V76JQomes8UTwIf7QRiKEEELSj3S0jyKyN48cOYJnn30WH3/8MQYOHIiCAv8xzEcffTQqmUtmxDNEOgnV3v+tVr2y22EQr5pwnVQN+qixBsPTR1bSkhVw6qDHsZDYM+YOpMe7Tw0ClxZX4yZle/+q8T1EzFA73VkIbPhW3iCyfw50r5MK02/qPvnQY89IIw12U6lTr6T+WwLYByNAsQgWedDrLNSxzDi9mTXlOOZpPHKOYMcUzx4RiwqVY4qXkBzfqoPaAOv2VgS9rs3nlesmxzL/bj6n+bd6i9+qld/NXkNWqMKY5EPuA1XUEwEnB0YdWok84by6hFD5ykJgzCWZ/qbWVb3yv5oH6fAHExGCiWAqORbpzNdAPHvkGkjdy1Lncv3k/lZX/QsW0BnQ7xkpq3k6rJ37IQv6vVaplEHyKPmV49TCqDPxjCmG3l6zlN/NiGCilkk8jtSpiuLBpa4OZodcWK886Ia/V5wg7VHuC3NML0C/Pg1KOjU/6nNM4rwlAqejWaky8hVPaCMRQggh6YcTGylV7CNHA5s///wzPB4P1qxZgwEDBqCoqAg//fQTVqxY4fusXLkyRllNPqxWfrGazuAkbk24jpY6gq52RmSql3TyJAiyBDu1OoYaY0dNo5bJyqNAOnLqCHe4EXlzOumghdpH9XAKdUNpShqp/1wY10SOY2e0PVo3bijvKC3M7+pNKZ1js9dPsHwG85YIVtd2HgCa6a+6r9X55BqoeQ3lLRVMgLPaT52K4+S+CnbuYNg9diwf9Fb1ayUiuZS05imaLtPfYPVqdW7zvub/w+1vFkvU31RUgU3S5yh5CPbitfISE6HFfN5I4ocF89BUnydm1GenVRs1lzWY95Hd4NixINaeQYsWLcL48ePRvn17uFwuzJs3L2T6Tz/9FC6XK+BTVlbm8MyxhzYSIYQQkr6ko33kyDOoR48e2LlzJxYuXAgA+PWvf42//e1vaNOmjaOTNjWcNoZKBI7Ayyiy2sHOB7Bf+S5xhFRPAas4IIXec6ixOVzw9yySY6qj7m4Y3hISg8btTV+J0EhHR8oh3kbyXR0plw5rFYybqcZ7LlVsk/xIJ7hO2SbxV5pBj0my3fubenNKp85uDJRQBBNpzB4VEmcqy/v/YeV32Vf1lDF7zYQTH2Q6mOqZJKt7Wa1OpIqFVqLbYQR21OV7EfTr3qD8ngN/b5n9pn3MeBDoTSf5EE80lSPePKmeH+qxzOeRNl1kcZ5QdSntU9rlERheUHJcK++sbBhtMpjoEuy85vqX62gl5qoeVIAufkrcIck/lP1rYHjIicgb6rmkTuVS09l9lklbt6IU+qp68iyR+xwwrpELRvwj9XmmogrLVt5t6vNE0gPhyy6ocY1EXFafj1mwfnY0wN8Lz+wBBOjPbivvtkwYMbSsvJLiQaw9g6qqqnD88cfj0ksvxfnnn297v3Xr1qG4uNj3vXXr1g7PHHtoIxFCCCHpSyw9gxJlHzkSgzTNv1jvv/8+qqoau0B1+uO0MdTDmL4jSMdQ7ZCr0whk6pZ55TArrwsZdVc7MupIt5pfsyCl5icb+lSQI7CeHmeFdOCkc232OlA9gdRVqI4gMKC1OpVHyirTRuTYudCFAA2BU6JcsBYVIiGYGGT2AshQ/qqeD+bpJlb/22lHZg8IOU8drNuCeal38znMbUTNRw4CvSfM18ROh9Y8HSuUN4i0C6vg5sGmEgG6EGi3jQJ6vUunXI6h1pU6fc7qfKHyFAw7sY7MiFgiD/IjynZ1f7k35D4J53EXrN05eQEG88bJh794rAqealsQYTdUPs1TCQURo6zavF2PG7XtS/2qgmAwzzgR39R05vssG9b7ynURz85EIEKsk/QAUFFR4bfd7XbD7XYHpD/77LNx9tlnO85X69at0axZM8f7xRPaSIQQQkj64sRGShX7KKIA0oLZ8EkX7K7+ZZdQRr0qwoS6GObpaIfhLwapyzWH6+zUKsdRV56yEkVU0UJG+rOhd8abAWgLPT5PKNT6PKKcuwH+ng+qSKIiHaR803bpoJnFiVwYsVMOw/B+qYfhYQQEL3M4zO1DRvPzYawGJ3mT1ZtEOBBvKxGsNBhTuKzEBdU7IZwarXrUCCJqqG3QnP8c6KJeCfRYLbnebSL2NfNuL/WWLRtAK2/ezGqyuT6bhcgvoF9rcxopg7rKmKCKieZyWN0/kh8nsWIED/xX3wIMjzh1uo9KsMDgKnLfBhOOpE6DPTeCeZ+FQ+49qUOr1eUEq+3BpkaZEdG1OkjegolEan4qoD+nshHc1VYEE/U38+pegppOxNdQMXmsVtWT9BKDzex1qcYIUrv/5rLmevNjVZ+ycqKsPJgItAg+ANCpUyeUlJT4PjNmzIhqvvr374927drhzDPPxJdffhnVY8eKdLWRCCH+iBd2sJVvCSHpQTraR448g2QumnlbumEeyW0soQQH9aURribVEXLpKFmdI9xx1Ckg6lSUUAGfAaPznek9dyH0pcOrLfYzH0MVgGDxvzm9inRe3fCvAyuvFClXhTdtHYyOu3gjFcLoyEXSes3tQ4wAmTKXB0Poylb+h5IP+YjH0hHldxVVEAgnBFl5KojniNo+zNNWROAohBG4WTq8bu/2BhjLmB+BLg6VIdA7zdzWi6AvJR4q34UADpqOA4t8q/t4YEzVkjYhHiJqG7Fqd3YRoc7sISKeH1b5CueJIsesQ2DbU9u1eTnzaCBioxpzKZiQJG1JbU924zRJ/kWENXv2BDuG3Ace6CJuBvQ2p8Y5Mnstmu9hNfi6ut3sMSlimFwHK9HRnDfzNDQ1SLS6j4ZAbzeVHOhikXhoqhyB0T6i+Q5ygtN57pJ227Ztfm7KVqNekdCuXTvMmjULJ554Impra/HMM89gxIgRWLJkCQYMGBCVc0SLpmIjEUL8kYFMGcwK5dFKCEldnNhIqWIfOZ4mdskll/gKUVNTgyuuuCJgpYw333zTyWGTjmgb4aE6XuoUDzf0ToLaCbPy9pDt5ik45mleVqvZAEZnycoDSsxWETFULwgRJo5AF4D2IPTKWIIaU0Z9ORbAfwRdRu/lpSrHFUHhoGl/6ZCrK50B/h4A9TDi3jQg0NNDFRJkn1Av8FLodbMXukeU3NqZ0IWxfd7veQB+UY4pSLyWBvh7rXhgrO4GGLF3JK5ROCHIPE1IRe0cu6HXp4hHhwC09pZDnWJU6y1fHfTrpHZ0ZQpOofd38XQwPyA17/nM03jMQkOoqUqqF4vablUPM9UzTMQ4s0Bp1TkPNtVLkHvWLAYdVv5Xg2UHO44IfoAhkgDWMcGsBDCpP1WQMD+jQokPgLG6mORHRD9VRFPTBvN8UrcFi5kj57LqAmdAr79gUxbVfByB0X7UZ6Bct2LosYfUPMmobCivP7OwJLHEVOHV/EyTVRcFVbhSzy3iZA6Ma2Iuq/oclbxIvgBDZLJa0S4eRCoGFRcX+xk70eKYY47BMccc4/s+ePBgbNy4EX/961/xwgsvRP18jaGp2EiERILqFWoW15MBsT0zTR+xH9VBGhf8PYEknaQ1x5gjhKQHkYhByW4fORKDJk+e7Pf94osvdrJ7yhDtl5O8AK2OmwljhFg616rngTk4qooqBlnF/Al2XnXFoWDTcFQxQl1qXvZv8J5fAtOGwuyFJMjS8YLE+ZGXrLxYiwDshr/3CGAtBpnLK2IQYAgFVqP5at2FenmLGHQQQBsAnWB4abWCLpxIcO5dCOwQS6de7axKJzgPRudPgmWHE4OkvkS8s0qnGi95MKaCyRScVgDae/ObDV0g8gDoAD0AtEwbU+tRPIkOwPCEsmqnVmKQdOalnoPdb6pIJ15X6tLk5uskecyHf+BrwLpTLQKTVZ2pBqsqKmTDiHejBpFW27bV9VLbnBoLS82LiF/mcpnFIKuyOxWD1Lya45O54S+MBGt/Zq8p9VySbzMZMJZXV5Fyq/kQ8dYsBokQUwLdA9AsBpnblxXqPiIGqW3BXF7zlDCrqYIirkussmBiUC0M0c+qcyQimBqwP56EE8St0sebk08+GV988UUCzhyapmIjEeIUsesEVZRPBg8asaGzvZ8c6M9gsZPqoNuc8kzOUNLJFOcG5SPvB4pBhKQXTp5ZqWIfORKDZs+e7ejgRH9BSMdeOnXqNB3pSMoos3RoZaqAGtNHFXnqELiCl0x7EMyNMBtGR1REgYPe4xV4/8+HIcpo8A/yrLq+yos83HQGVQjwmLapXglqZ1gEEBlltxKb5MUtozLqS1ftoObDCD4t4la2KQ+SN1kdTaZwyCiQlDkbegc0F7oIku/drwKGJ0QZDK+UYAJJBgJXCQP8DSWpd9XwCPUAMivVpQBaQDdeymHEBcqHLv7kedPv8/4tgy4Cubz71EFvDwdgxGISkbIeukhU5D2PPEQOI7BjK3k2T+FSf5eVu3LgP/1KVl2TdhjMK0atA8D/HrDyelE9c6S9mY+peuapYogqpKriWzjDNpQHkpovq3tWng8N8Pc+0aBfRztButX2Zp5eZxZZ1PsfFn8FsxCklkEVtdT91BhkgrnNQPkux9NM6dX8qf/bnV4lQpU8a8zPJhWZGqlibquA8awIFhNJkDpX/4qAJchqg8SalStXol27donORgC0kUhTRuJJSoB81YNXxBPze0dElnh2msTzUp7/qj0p283xK8XGUt9HMqinTkdPRo8nQkjTIRL7yJEYROwjnYFMGNNW5CWidsbEo0U6MeItI6PzcoHMMWpq4N8ZEW+eUAGHRRCROC3toQsAOdC9XCqgix0H4S8GycxGEbTUl3uokWt5uarTtgD/kXsRsMTLQF7SLhirqpmDwkpnU7yq5Lt02FQPoxIYwk0VjNg+cl5VqBLvrFrv38PwF4Ny4e/9U+Qtw17oU+Y8ADZ795OlqK2m2wSrNymnKpDItEERioJNfTF7VbQDcJw3bz9CF21aesvSDob3zDbvMbcp+aqFXodl0MUiEWRkHzf0ttICevDnAujtTq63tGv1I0Kk2cvDA2OpeDmOIMHAzavVqddERQQj1bBUBVLxjFKXXRfjL5SgorZ5OUcW/O9JaT/BDNpw8X/yoJfRvL/EzJFrIqKpTJVT41+FwjyaqeZbyiH3oFxLK3FGxRzgWJ0uaBY8BasAm8FW9FM9JM2Cknr95XnigXUsJyvk/Nne/WthCL7mwN3mqXvSjiqU7/KslPZv5RUlmMVQNT+CjDgnglh7BlVWVmLDhg2+75s2bcLKlStRWlqKzp0744477sCOHTvw/PPPAwBmzpyJbt26oU+fPqipqcEzzzyDBQsW4MMPP3R4ZkJILHFDtzPy4e89DhgeN6oY1ABj8CmeHjRil8tAl/pRwzqowo7kV93eoGyXj/kdx9hBhKQXsfQMSpR9RDEoQswPeNW7QF4wIs6YXyBqx02mJ8hLMU/ZV/WmkO/SMVddUgGjM6IinaZC7/6lMDrXJUqe1OPISmOZyjHUzrs6Gq/+tWrw0ikK5hWiekLI8VXvB035K+VXl5gXjyvxaKiFXn8tYHi5mEf6rQLGioGSC8NbSrarZdNgdAAPwRB8DnvPUQ5jZEwCVasBbVVRx9y5NW+Ta5wPXcwy16+5DJJ/ER1lvroan6gKuuByCHpbO+T91MEI2CujXNKxVq+vGGwivomQeBiGWOGBMTVMRgfdMASUbFh7VKieYtJGVOPKjHTcxRBT60O93qpwYNWpV8UQq9hecmz1u5wjnFgSCrM3UjBR1TyNTcqjXhcrb5ZgqPcUlL9u5f9wni1mROhTR3qDvSyDxe2yQkRvta7V6WQyFVDaYzjM3k5SD+rIbrC2Zs6XKohLnsxt17yPeo3McYWsvJESRaQxg+yybNkyjBw50vf9pptuAqBPsZozZw527tyJrVu3+n6vq6vDzTffjB07diA/Px/HHXccPv74Y79jEEICUb1b1GefU8HXDtkwplVJx0J9V6n2oORDHdxwKgap73Ynz6As+K9cqdaHaoeqdolME1O3Af4Lf6j2tPm4hJD0wYmNlCr2kUvj2qeoqKhASUmJo33UeCGA3smWzklz6B3z5tA9RaQjHawBiZiRCX1UpQa6WFMGY/pSHozgv4egdyJqENqjQeLjdIYukLSDHhfmIHRhaC+A9dBfqK0A7IARR6UIhhgg3i1q3rOV7+bf5AUty5OL0KXGGFE9N0TQkRepeEVJmmO9+c6BLsa09e6TD90jp8j7ey30ODftAWyAMYK/Hfq1qPKmMQdbbgbDM6U7gGUwvId2K2XL89ZTCwA/wLhWIsAc8dZxGwBboAtLDd7zS9lylXKJB4a6mtce728SQ6gHgO9M9QoYHXAxtrKgB4Le6T3PUQB6en9b6s2LBqCj97xHoHv9bDMdV84j4p14hYkxJFMKq7111h56OxJRMwN625J0UofVMMSt3RblAfRrdBj+QRdzoF+f3QikGEZMKPFKMk9xKoThXSUBpuXY5gDI4iVi9riz8haBKY3Th6h5epcYpuFeHMUw2k0NjNX8nMaWMQtfLaGLmfUwvO3sLG3u8uapAnpbkWdGqPo4GvqzTaa5qoHTVUTstQrcLOcWISjcNTLvqwo36rUogREzKxhub55F/BXPThF35Lkj00Xl/jRP1WsG65X2XNDvITeMIPQAUF5eHpMAhIK8B6fBENztUAPgPsQ+fyQxRGIfkfgS7JmbA0N8N3u3BIsxGAniVd3c+8mG/iw8BOPZqsbgUeOrHYZuv1XAGap9KZ6ddvKpegS5lO2SJ7FXxRYRnNaVHD/aq4ISQgKJh/0RiY2UKvYRPYNsIiMJqgcD4B/XRu2MiKeK6hETDLMrqnTE1NEK6ZxIx0m8e2pgBHuW/MlUmGbefBfBEFiOQH/JVUHvjEn+pDNpNcXBauRdHakP5RVk9nYJ5k2h/iZ5Un9XRZQs6AZHDgxha7d3mzotSOZ+y+iTB4aHkQQkPgxdRNkH3ZiR89YjMGaHpK+GIY6IR4TkX2KuyAiSCGnqCJiIKlJP6px1OX8O9E5zM+V36bSKV0wDDFEoG0B7F7Bf0zvnddA9iuS84r0kxlk9/KcuWiHXPcO0TY0xUw9/zyeZHpSlbBOjqFD5Xz2ndPrFq0hEG/HiEgHUPGVInRqntk/Vc8Mc1Fn1pFH/l/3NIkCwUcdwK5EJwe6PLNNvwe6hcN5HagBrtS6k3u0GUTaXM5gYonpZSRr1WWiVR5jyonrfhJtOZfbyMv8eLr/m9GavKiC8F4+gjkKbR4TD5cH8/FQ7CWqe1JFoOysJxgKno9lNfjSJxJVw3sjBfktnxD5VPSWzYQwiumC8q2WwRWIimgfy1Fg6gL/Xi/nZoG6Xd3Y1DDFIpoCpaSU8gOo17uR6SXzBQm/5NOg2TR0CvXdUbyQRfMQTXBXD1HfSEQR6AEWCvCetnuGq3aduU+uaEJKcOLGRUuVephhkkyIYLzi1gylBngtheLjIlKEqGNOaQj3gVQ8bERIOwuhky1SZHOheKfIyPwp6h7/Um15Go7tAF0d6QzcEJCh0GfTRl+3e/Q7C6JCId4cYDmpjt+pYyotUXQFL6iTYtByrzpJVh0y8E8SLQ4PuubAPhgAy0JtuL4DjAfwE4AToI+m7lLLkevMoIk0b72/ydzOAkwG86603mdZXDd2zRRXz6rx5EIFHxDQRgbJgeDtUePMpwpG0gRxvGjUeiBhe6spbzQBcnAlsBPBVg56mA4BN3rT53vOWeOugGMCoLKCsHhgM4H3o3ksyLbAchtEkU2+CdV7N7VSmw0ksHjGUmsGYIiYPkhzo7V72kelrVdA91CpgeGhIWcUr5Qh0ry9R3Hd79zvKe97tMNqEy1u3qgs6vMeU80HJs4hnHm/dyYpWIrBlwF9cUrEa2cuDUY+A9b0tBqjVCKycN1j8IxesY+moAjFgeByqS51Lp0CtBzNyPrkG8kyRfa1EJDWwu0yBVYVx9ZoK8j0XhiAp3jRSxmDt0BxvS4K6q50e+T/U6KvUr3RC1KmbQGDQffFgNHcsRfCXZzTgf171mFbij0oGjJFwEailTUue5HkQz1gaAMUgktxIPBbVy1OQ95Adr8Z0Qbxy1DiFbui2QSmMxSKqlI/qWa561WRDt3UL4B8EWhVaVFFIxCUNhofqIRiDQqqHtwhU4q0kMYQkz3bJhW4zdPT+zfSec6/3UwHjXSN2s9iCcr5qGIthAMYUdhHN1Hd7pIgQpb7HVXFKjVGkTllT80UIST4oBjVhpAOmehSoYokqEMnUBnWEN1iDMHcS5KVhNnJkNFrECnWUW7wpxP22EEbcG/FgqYIRH0a8lqyEGnXOtFo+Nf9Woxnqd7UDHO5GsOoEm7fVQxc7pLMreZdOaT4MI0ZGp+Rlq8HwisqAMQ2kDXRBTDUQxGtKdac250nqU665uY5UjzC10yjXSzrr0uFUO/fq9cgH0CYL2HJEv0nbQRdKfvHmWcSl1tCvdQkMDzCZniYGheptIJ3ZYN4bZtFSbfeC/K8usy6CoQRjlvo0ezyoZRakHatTeMwdcav8qkam1K85rdV5XBa/q1Om1PShOuPSKQllyAZr3+ZzmaesWXnoqcdT22OougmFWg9qGUM9q6Q+VCFH9QgyP89CoYpl5u1W9abe0+HqPRzBymj3JR9M/AsmeJvTq23Aql0C/p3eeBJKKA6WnpBYYRVwXn3Wq89wdVDAjqdeMI8NhNk/GZHnpupprHpGy3R0sXfUVbPkmeqGLh7lwhAxVC8Z80dFbCKxtcxTfAH/Z51qWzitaxF58mFM6Zc8yiIhMnAgZRbRXfKuDizJQEgknkrBUL3j7XgHWdkghJDkw8kzK1XuaYpBNpARW7UBqCso1UCP9SJiTBX8PUryYHhmmJELIJ2qauidetXbRrxyqgCshTFl6iCM2BNqINNO3t/2wFipaSeMWEAagnsMyMi0+WWvNmg1/oX6YgeMl6+IIiqhPIPUsqpBjAHdG6cn9LLXQ/cQ+R7GUu8F0GP5dADwFXSxpyX0eqqELqLs9R73OACrAJwF4L/QRaFD3nJvhj6qJCNllTBe6GreRTBSY88AxiiZiEEVMASjWu9xxGumLfRrUg1jqtlBpT46AuhRAny5X7++k6CLfDuhe9js96Y7E7pAdCKAknp9qlwrGKN5R2CMjJlHAVVkGpe4W8s2CQytdlRF0BFvDyHfe4xy6PdEhXdbhjfdHhgeQFDqT6a+1cKILSTxV454j7MPhohlFmLV+EkyJc6MxGsye+lIG7UybnMQ/N6tgTF6us/ifID/NCrZX+LqVMAQij3Qr9seZV9x1zfHwTHXndSJlbBjHl3MNP2uxrg5DH/vIkmvik6qV41M36v2lkWeA1bT7NR8SidBnotQ8iDp5dlqJZhJO86DLubaQb12qlBvhYjN6n7y/LUK0K8KOnnQ72+XRRp12pfVSLh6DSXAaT4CvZjiAT2DSLKQBf15kAfj+dKgfNQBFzXuizooZ0Y8MtTBCdknR/kt3sudR4oMJKkihyzXXgNjSjKgl8+jfFcHvPKhexI1h14H1dBtkkrvxzz4hSD/11tsF+SayDPfnA87qM/lfOi2XhZ0z+5K6PZcJQwPG/nIYGqdss08CBpORHSCiFBqPEB1QNksBknbZowhQpIbJzZSKrxDAIpBthFxQn0ZqoKPTBMCAqeOhBrhVWOWqJ1GdYRCBBINRuBeVZBRX2iqu6nEBlIFKpljHeyFZ3fqUDDMXhfhPAXMIyZmDwnACK6tTo2RgLXSOZTfxWtHAlzXQxcBRGDI9Z5TnQIm072qYAgTQHADJZi3inm/BtM2yaO6Wpzsp3oGaTCub6Wm57UtDOGvpTev+dDFsFwYgZJl6iLg38GWUcFQo16qR466r+TdPH3QLMpA+V3uCdVrpBaB4gbgL0AcUdJI/YlAFa4NBhuBU/Nn/j3UMdURaDPSdoKdT0WtU/lfbeMikphRj2uu+3BGueTRTr4krXklLbMHoDlvavmtvHnUPKmiifo32GhpsOOFijGkHiPUNQn1m/n+MHs6ms8T7HuG8lc1/oPl02Pxu9U54kGo90Ow9ITEAhlcUg1VVcBQn0HBnilmzN4Y6vvM7H0Yb6+8SFGFBvF2kYEpESTU96z5OSyDD+JhLsdUhRS7zwQ77+nGPDPEVpJ8ib2kCoFSHlnQQOpFU/5Xy6M+8xqbP8Fl+qiodpJ6/mh5JRFCYocTGylV7mc7tnXS8Oc//xkulws33HCDb1tNTQ2uvvpqtGjRAoWFhbjggguwa9euqJ5XOsTqRRVPIbVTJ4KQC/7BhyVujRXqMuqy0kIt9E6+CAb58B8Rk/PISMcW6J4vZd70P3m/b4Mxh1qdYhVJ4zSPrgPBXWrrTL+rIhVgvBhFJFNf4iUwpqCIgSdGmWqkbPN+NgNYCF3QWQMj1k9z6N4z+6ALKIA+1Urmte+EXm9lAL71nmsfDM8oKYdce0FEJRUxLMzKqrnOZGl2WR0uU/lNFV080Ee33j0AfN4AHAOgfbZ+rUdA9wLKgx4byAWgqzdf73jr4RtvXTWDXp8SG6cURv1KWcz5lTzLCiQiEMnIoYoqanm8594L41qJd1i5N/0+GF47cmzAGMXL8Za7DPq1lelmG+EvIpmnXUp79CDw2qhlq4F/zC/z74KUXWJtmVFFPPGwE+PTjAuGGKm+QFTDHQj0ZpLRU0HafrB7TtqQlYeQ+l3dJu3PKt9qR0gdcZbz1MIYLZYpBaGOZ56GJtepTtkmHTBZmcscwF3iI9XBiEsh10JNWwhnIoo6cm72ZhSPH/W7lNOt/C/tS+4XEZ2LlP3EK0javsTGEpFUAqSbp6zG+yWtRfAhiSdR9lGkqFN41HtQyIJ+j8izQhY9qIT+3K2G4f0q7xp5xsu71uqcYneYpxC5lY98NwsMTssXr3u3FkYogGrv393Q36NboHtT74LuTVkJw0vYvAiGeNYe8O6/B4atmyzUQbczNkG3dddDXwm3HIYNJfF/JERCBXQvp4MwvL/NmD3NGoOVCCT1qw6W2WmzhJDkIh3to5QRg5YuXYqnnnoKxx13nN/2G2+8Ee+88w5ee+01fPbZZ/jll19w/vnnR3SOYJ1kK8zeP/K/KmAIquARDOkISWdPNY7M+0oHQV4uYhTJC+4Q/FdyUDu0wV42obwA1O3mj9ngUUf0wx3T/MIUEc1cf+LyraY7DGNlrz3Qy3gQ/nPh5QWrxg6SldgOwhAhDniPGyyOkjnPViKKnfKqo3JWxq/ama8FsLVeN3pkefdq6CKXdE5LvOnc0Mu/w/t9LwyXdzmPGEjmjro6mqqKDBnK7+q1NudX3aZODZTROLl2gNGZN5dbxLQM+BtIkh/xaguGOroW6t5VDbFQqPed1f2i5t8c58cqrZqnYOe2Kp+6Tb0eoVDvJ6cvoWDp1fpVPVnUgNXqecN5OUk7txpdkXZj1ZFSR07V5wFM/9t1dw11j8rvVqO75r/mfdVrHuxZof4m9WoembYS+OKBJ4IPSSzxsI+iTSjPCSDwWSYiqXxUb231HnLiXaHey+rgR4byCZXHUMeNZL9Ikeex1Iu8R6thBI0W0UHSyEetM/EGUoNLJ5tAIYMRVdAFoHIYq6SKoCJ1IEGs1Y8slmB13Gh6BVn9bz6feepYqnQcCWnKpKN9lBLTxCorK3HRRRfhn//8J+6//37f9vLycvzrX//C3LlzcfrppwMAZs+ejWOPPRZff/01Tj31VMvj1dbWorbWGBuoqKgAoMdp2QE95oqdsTPz/OIG6C8lVRSSZd+DvVDV5efroHsjSGdY4myYp9bUQvd+EUNFOg4iDNXCf+TD/L/VvGgRS8QrSQQmwOgUZ3nTtYIupNRDD2B8GIZXkoyI18AQYaxcctX/pROfpXzPhTFnP9N7fClHIYwRIPGO8kCPnSPnLPPmLxO6OCIj98u95ZoPI6ZPqJtV9UwCAj021PKowoCs7CYUwIhB5PGWQa0HKbsYDlugezbVQY+V9P4RfXWwPt5y7gWwFcAy6GLWL9BH8sSTQZabPwRjxTeroN7q/9XKdzFOMmFtJIkglwcjiLTsBxju2a2hj04KGdA9lqTuVcxGvKz6ZcbcYVf3sRrxM5MLvX7UOFvqMVUxQp2eJqtZCer8fk35X20zGvy9flTvQbnnM2G0Fzm35EcEA3XVE/UFYzYexXPNPPVO8p8BwzuwzvS72XtHVqJT234eAr155F6XWEL5CHxmSZ5cym/qc1G9jhLnSkXzll0E0YMwvGyk7mW54hz414t6neVcIjyr9wVgxJ+SfWSkvJn3HIeU88qzW5AV5lRh9ZD3N7Vu1Y6XFSLayjM23h0Ep50SdmASS7zso2giz1exD6za3BEYIobci+bp+k4RsUNFFacBY/DNA8PjUfIaynvD7OUc7869VcdDvH7kmSLPSPESr1X2K4dRZgkxEI/8m9/hdhDvaWkje2HE8PNAbzPiPSYDgKpAFGtUDzSxo5JNVCOERIaTZ3uq2Ecp4Rl09dVXY9y4cRg1apTf9m+//Rb19fV+23v16oXOnTtj8eLFQY83Y8YMlJSU+D6dOnUCoHfQZVpWsNFfFavfzMHzZNpPMMNfnYohggvg3wlTO5myTzb8zy/Hl3ni6ovH3BEJ58EgqzMI0jESj5sCGKtwFUGvNzlGprJvOC8F9YaS80s5VG+oDBhTUgBjDry8aOUFLzFzNBhxgFwwOtn10AWTBuhiixh2dj1FhGDXUi2LLAst5Ci/WU1nMt+Ih6ALPBp0Q2ejBvwMvRO8GUZwx83etDtgjALK9BLpAGfBcIEOZZCY25l5ZFTdJtPAgk2NE6O9AP71pwqGZswP2GBKtVmUsroPQqF62Vg9AM0Cr7qfeq5gwpo6wqyKRHJsddRZ0queYuZRRdXLSr0mwVBXiTGXVb2vtCB/1eOYy2vl4ah25kRkMaPmQ+rUqlMndWZVRvEccyPwuSL1Y155SI5pVaeqZ45gnhYi10+mj6jpjyB4e5U2ZBbOoHwP1lal7IkQggB6BqUa8bKPool6/wYTd+TeU72c7byvw2Fuu5IH1aNEziPPTBFRQj13JW2mjbSxwKqDInaPGjTZ7F0l5a+DYUPIwhaxpjFT6eqh23kHoU9B36N89kIfTDvoTSODpOFsoGigvv/teL8RQlKLdLSPkt4z6OWXX8by5cuxdOnSgN/KysqQk5ODZs2a+W1v06YNysrKgh7zjjvuwE033eT7XlFRgU6dOvlWpsqGsVKSueOZDyNOiCpcyAsmD0asExlJEkPGjIxCCzL6rW7XlP/Vc6sd/TrTMVSvArMXBWB4K6mIl438L6PcRdBH4jO8fz3QY89kQH/RHu3Nw88wRmTUOBpyfnNZ1bxpyu8HYYg5gsdbbrf3/yz4ByOuh37dxMCRTpQEYS6DYexkefO/BeFR4xaZEYFFOshF3jwWe/PQDbpIs8ubrgX8V3CTETjpcMpDw9xOmkFvTz9A9wD6Gobg8xX0+nLB3xNE2t8BGJ4KR5R0MmVMYpSoZSqG7hl2CEbMhP3QA1hXQV+xQwPQHcAG6IKVGVmxRMQg1dtFjG6zECh5VLeZhQfVwJL95TqYDbxQo41SbhFgzUgbk2sv7UhdfQQw7nWr86jPBGkjUge5sG4LOfBfwUbKpnZGZMlcyYe5nNKG1ADqwhEY93UD9Ot8WCmbinj3mctSDSNemQjnqmgh3onmfKlxxg7CuJflXOJJJW1AHaWWc+d782vV0ZL2kgP/e0jqMEP5TdqO1TO5EMZ1lvOI6J0BvZMBGG1D9RqU/XJgeLS5EdhuoKRXKYV+z4pXn5WoFg/oGZQ6xNM+iiYiwiRT25HnpSpUybNXQ/D8ShpZTVb1XlLfV7FERCizQC2oeVeFLxWx31wWv0ULeU6LDSLPuMMItEfsIJ6b4rWpepup0+CEWLc5eV9I3CkZNJNzqwKk2WOVEJIapKNnUFKLQdu2bcP111+Pjz76CLm5ueF3sInb7Ybb7Q7YngvDoyMb1ktU58DoyMlFVjsEsp+8gIJ5ZMhL0SwGqZ1GQfZ1I7ATqcboUbdLvqw6IVaj+6oYJB0n6ew195arjfd4soR4HvTl3AuU/MvULHNezGWVUX5BXopS57L0qJRNhBx1JEmEjQb4L5kqYoMYGhXQr61MyRGBIhxqnVj9Ji91WeJUlhuvhb7ilyxLXg9jmWjVqDQbilbiRL43/zu9v/3sLUutd7ugHkfquRr+0+lkipFMyTOT4c1zC2966QAf8m7zQA/C7YE+XW0bAo1nqRvxZnErZZU0ZoHSZbFd8gNTWlWgk7p3qryrebbaVz2+ep+ar43ZO88q7+px5P9s07HU86jeQXLdVAPa7K1j7mhIG5KnpVo+EVeEfBj3ipUYJKKE+nyrgyFIiThl3t9KZBFBxgX9HlfFIDmf5B8wOiVqft2mtGbkuR3sOaMK58GEQPE6Ul/2mfD3DDKfU6aNqc8DmWQjYpRVOzOXvwC6UCadtUThdDQrVUa+0o1420fRJNmEICD4M8HKm1JF3qfqMuLxLls4j1ErL1DzfSsDMtHE/H4SAV0EE2llMv0+knqzGsxIJPKeVlc4k7alLn6hegwl271ACAmOExspVeyjpBaDvv32W+zevRsDBgzwbWtoaMCiRYvwj3/8A/Pnz0ddXR0OHjzoN/q1a9cutG3b1vH5tkL3SFGXtzZj9dJRDQjz6EYwT50S6J2xMugd8JbQR4VlJSPppElnMAvGamNHoIsMGRb5MXfOzWIDYHgmFED3BMn15qXSe57m3nwUwVh9Sjo1tdCnWhVDFyNyoHuNCB5vGvMS8WodSflUIcvqZeiGfh06whj1Ea8NCQYt/0tHWlbq2q+UXUb8GmAsRR/uBSz1nAvjmh3xlrsaujgiXl+yxLvEFtG823K8f+V6qSNy6jKuoYI0inhwBIaIYNVxtyqP5Fs8E8T4knLAm+dC+K/IlQdjKmAB9Dbqhu65UABDHKmCvpJZN28ZNkNftUTtfItbuuomrU6B1GBcP1Xo1OAfL0gESg+M0U/p6AcTdADrufp2DK8MGLGorO75TPh78UhblU6OCJNA4JRLEQrEQ088YUpgtI96GJ4sqmeVGJQSu0ctizqCXQf/+EGyXc2zTAWw6gB4TH9V1JHOWqUuBNWbCkpaqcsiGIa/CD6ql5DcFxqMexUwnslWcTvkWSnTZKWToZZNvD2tOijy/bCSnzzobVI8tszPdnWqmflZLM9Tee6ocUjMHnGSf5nmCkRvVZtIoBiUGsTbPmqKqLFf1OcwYHT6ZaEGeRao73n1fo8l6oCDGXkPSD6BwOduNBHvGBm4EFtW9d50wwg10ABD/E91UUTec2KfiM2iDmyJICTvBPMiL4SQ5IZiUJw544wzsHr1ar9tU6ZMQa9evXDbbbehU6dOyM7OxieffIILLrgAALBu3Tps3boVgwYNcny+X6C/lKph7T0jo+Vm1IttJSKpHR154UmsnT0wPGxkSeUsGFN9zGKQiDKHYEyNUpGOh+qRAPi/ZHNgeMi0gd4JLYI+7zrfe55i6C/qAujCingp1EKvJ5lXLlPq1LKK6FJr2m7u/ISbKy4d2TYwRt8OwRiplxF5cc8WYyMf+vQsdVRGXfnIyvAwf5fOrHTIxQOgyLu9OYwpK0Xez37o9SaClAtGMOf9St2Ih4YYTOZOvZonEb9kH1UgVNNZGYFyTOk4i+ePBCBv8J67EMZ8ehFnpB2UKHlt5t1frsth6GLQcBiBdXfDuEYiCJpjC1nFWpGOt4rqmSejiWrQXtUlPxihxKBg7U/qU9qvlVBnvm6qsCPnkOOoRncGjGeEiHwadOGgPfQ6kGshnkfy4lFFVGn/6v2veqGIB4+U30oMkpVlrESHUCKt3HciBklbEtSYN+ZjNkBvW/u921QPKCEbhteSTDmDcrxg94uIjPUw7j/1eRzOswgwng2SXsQgdVqqoHouZZp+F8FdFS7NHkeqxxXgL3wFuy7xwKnXRqp34FKVeNtHTRERfK08b1SRRZ4D5pW64nVvqLaeGdVzSQYMgNDTyhqDWi/yzpL3l9ixedCfkTLlXw1sneqoz/cs5X/5TQZB1QECikGEpA5ObKRUsY+SWgwqKipC3759/bYVFBSgRYsWvu1Tp07FTTfdhNLSUhQXF+Paa6/FoEGDgq6UEQpzoFczwdyIzbSAMZVnn3Iszbu9CPqKXIDubeGCEVRXOsCyoo4a1Nfj3VdEmhwYq27BlE5tgDLKXuzdryt08akFdI8kmRohHdUG6J4Ku7y/7YfxYquHsYqDdMrUDqZZgFDzpcE/poe6j9mLCTA6/hkwPFkKvHmTjnIBdDFDPApylXM3gx7jQ41PkOVNo55HBB1ZelVEEFktS0Q6mQLn8qaVpemzlTJVevc75D3/fgR2ygFj2VZV2FE9BgBjRFKusSpeyEpbVp1pMTbUfAFGuzwIY+paufc38R6qhhFPaR+MgJJbvdv2woiRVAnDSyUTuphRBt1jrLn3/N0BfAfDy0e8VgQR89Tv5k4yYAhiZoM31INWBBP12Gp68z0u4pVVnZoxG3DmYL9yXvUBq44auhHYBsuh140Y6LWmfQHDo0amcUq55KMKR+b7C8q+ElQUMNqZTMVUY4eJJ52cR/Iq53LB8FqT+jCLlWbUTpUIZep0T3NbNncS5HmXB+M6mGOQWNXBYeU3ObZqnMvzXUa0JZ2srCj5EqFbU9KICN+g/G9+Bpvbk1omVciTfAfzTo01FINSg3jbR00JGXzIh2GbyfNNFYDl2SGDTTJlKZ5CEBB6pFqdtqS+a+VZL+8qdbDSjiijPuNV20Xqxfx+knTi3Su2pgx+NEYIUvMerN5VT04rjy353W75g6Hur77rGmDYraoYJFOu00EII6QpQDEoCfnrX/+KjIwMXHDBBaitrcWYMWPwxBNPRHQs1Vi3uoAa7I3WtoGx/PF+ZR8PdOOinfdTAV2MqYARd6caxvSXQhjTRGT/ZtA757KCVwGsp6apf6Xj2db7OQpGYOhi77mkY78LupBRBd0DyAVDDFJfVhKv5gCsgzGb60leiBK7R8qjvpytxCB40zSD/jIt9J6vpTe9CGvl0DuSzaB7WwG60CYeCJL3bOidSHNHvASGN0iu9zi7vOergDENSI17Ug6j8yiCmIhoR6DX8x5vXqWzJ8iSuWIIAIHTqWS0TKa9qQZMAYwpLVbxmDwIDKbbBkY8k1oYgcCh7F8LPTB0K2/eD0BvGxvhL8ZIYNwsb16KAHSGLiCVQ586lgPgGG9dVip5V0UOEfFUY9IcU0fqwuzJBYQ2/HJhLO0t29T05n1FiBDhM1RnXDruYjyKMCrnEG8oEQHUqYaA/1QpwAjUbQ4ELvmUbeK5kgv/la/UqQwa/F9WZgFMngdqHC5p36poIZ0hEfKkbco+8twRDzpVDAqFaiDLs07uD7f3fGq9qMKciM+A4RkpHaFQQp9mOi6Uc6pCmuRBjuGC3nbVtiPedep5ZD8Rg8yx3FQvMDVP6u/qlMBEikEkfYimfdSUkPeuxEYEjPe/+b4UEVldrSveBOucyLtB3kXqKqbm6dqq8G5HmFAHBczeMPI+gulYIgSpQpF4XTZGDJFBulAinAh8gHUMQHnXmAcWBCvxywr1fST2jFqvqsew2AqhBk8IISTWpJwY9Omnn/p9z83NxeOPP47HH388bnmQjo+89GqhCysSb6gE/it/iVgA6C+gCugeFOJhUwtjaXB1ypU6bafWezzpvOS7gCxN99QQrxYPjKW8i2AETC705qEljM6dvLTFG+YAdBFjHwzvFvO0F7ujF+Z4QVJP9TCMAHX6jvyfYdpXOmAS30jiBKl1o660Jiso5UC/BmL4ZHh/lylXci55McvLWp3DLR4z1TBGsTzK98PwjxOgXt9CGMvZy5QrdXU0EfKkfOK5JdvE20KmuYjnlgeGmFavbBOkHsVLw+x6LEG5D3n3q/CeR4QsNYCxrJJl9uIBjIeGG3r7+wqGx5EIXCI4iVAgxo+UTVCFFPXY4p0h+4mRZw68LGlVAaNe+R8W6VXUTr5qkFl5IQlSDtWDSaawqcdVRyKlnUk+raYcmVf5E2NS/pfA0LI6oToFKQuBYoO0ITWfIiTVKt9VkdtqGoSUD/AXPySNx7SPCDwqatlUYU+tf5nuqRrOqreW+WWlCi9SL5nKb3J8MfDNxr96jVUBTfKkegKo08DqTfuox4Pym9kzSZ0ubG6LaodK9Y46EiR9LHEyH17Sk+QgGeyjVEF9RpvvMYlpI55Bck+IUGsVDy3clOVYIu8Hq/OLUCV2mFpW1e5R68JKYLc6p/kZqL5L1EFQ9dkn05slL40NAK16xqrTsazEHKsBEkH14A82GGwHqYM6GPEgZeq22JEajPdZvL3ICCGNw4mNlCr2UcqJQYnGBaNzL0GC66F7RWyCLgK0g/5SEu8ZtVNaDz2uyl7vMUq9+6yB/qLoCuPlIB22Qm/aA9BfoK0ANHPp2zdqQGvvMeu8v+V4jyPeMoXevOZ5j70fxouqBro4sBO6J0gVjMCnahBYES3MnVgrgcg81SMXhieBNDgRQOS40unJhSGciEfAARjeO+LpIWKFeFc08553i7ecIsKIgVAKY+pdrVIWF/xXkJNji0By0FsHIu4chCFyNFPKLnmuhy667YOxIleJ97vQAv6jcLnwn4ImxlgmjJWXMpRji1eRbJNOrhv+KzWp4oTk0QO97Uk8lDoYwblF3NKUelaXYRdRRryCxEtrOYwphyK2SUypKujXQ4xO8/QrEfsEaTsSh0FiDOyGvweUtB0RAqT+85Q04t6vngvwFwDU/JiFl2BikAhVEvTaLLAAhlGt3kO5MLxsZAqimqcC6PVuZbCKgVkM/ZqIWCv7ZsN/dTkov6vT5VwwnieSTxHv5HcxmmV0XPIugT6lMyT1p4oqElDVyptGnmtm8VPuIYkhocayyIURWF9EUdlPphlInDC5r1XhR7yeZEqmitkzUTp1km8pq5RJnn1qzDH12WcWKkWEUztAkiez8a92CGRqqnglmYNxx5pgnaFQ6QlJJcQukOlT6vQgGTwqhrHAhjzvZNCtBold8c+MeVBF3a56WKviOOA/AGSekhssNptg1clRRY94oYpY8jELLKo3ajCRK5hHkFOkDtRp/IAxNVute7HhUqXDSAhx9pxIFfuIYpAN1E6iLIlpXnZeOslV0DsvDdCFA3NDMHu9SIdG9ZJROxFioMhxpJMvHh2S3gV9GlA77+/NYHhnSNoa6C+ePTACBNdD73BUeMsk+VENI2n44abOqHVhRupQvIPUAN3ml6Mgo+sS30ZGW2QfcTEuh35dimF4R4kLrgSSbQb92ohQIg1fDCPz6JRa36rHgOpWLS7PLuV3uT7iUi55Nq8oJXFIAP8AvyICQflfflc76apBBxgxZOScVtPuRHRRywElrZVXh3pOyYOId9JGRaQph36dNOjiTS104Uk6tlD+qpinucnx1RgsItSZR/dU41UMQSHYUrVWXimA4WUCGMq/5EFNK/VlDoqtHluOrwYhNwdZVuMHqMcXQU194Ug5Rfwwe6FYTU1Qnx+qwKCOIJsNY9WINguK5n3kOaXmS61/s9eVOiXRPEItx1CnLpjLYNXRAfzrw9x25dhW7U4VXtVjqueX54j5XlPblvqMVPOVA/8YcOp51TyrdaR+1LqJJ+by2klPSCoh7zr1naE+Z1TPDsAQzFWPzFRC9Q5Sy2k1QCLvo2CDIcmI5Fkw5121r0J15KJ1XeU9It750qbUaeDyXkomUZEQEh4nNlKqvCsoBtlAPAHqoHeqiuEvBsn0oRLoQstP0DvG4r2iNgazt0YV/BtVNvw9cMxz1MVbpcqjizjqtLIBALoA+Nn7txr6dDRAfwnt8x7rFxiddhEm1A6r5FcCuorooXo2qJ1YQTqstQjsCMrv1TDmr0sMD6kTCbQsSOf8MHTvpz0wpjiJqFQOY/W3njC8ZwqhL3UuU+O6A9gGw0tAvDJkepN4sZgxz6lXO/11MLyd5BpJuSVfGd48SxwSePO6WzmfOjJUB+OayopSMhopnU7xnlDrrhhG7JX/z95/R0mWXee94O/e8BGZkba86+pqD4+GbQAEAUJsgp7gG4kjaehEzqJEcB4JvdGIWm+JlDRLXHqURHKJdokiIc4TRYp6dCIokBJAeEc0uoE2aFNd3mRW+szw5t7545yvzo5bUd2V1V1V3UDutaIy6sY1x5+9v/3tfcVisqF44MIA5QW1xqm+iwFhmQ8CJsTWyBHyVLX8c6f8b+dwDKEcLsdQiuuHDa5UzizoV2H0ld8TOEA1T2C7iPFjGRZipmTDs/TbGs+9YGcVMbGqpPCLtdVk1HOsED31U0rwAmpOCOxp4wBasbl0jt6o0iCATfI+K+TJMkns682zuaCGhHlpRX3Wx82fdYKBkw15Ejhj21LhkjY0wtJjbZ6bAaNrSlYEpEcEtph9vbxAID1HjCQLIo8DvCxIHTE6jlR/mxspOx7kmbVi2WfZBP3gxvsyo+2QBbEg7BN2nsrIsm8ptGt9wZyj+91sY2G73vGXi7KzIzsCoyxYGyKmtVLrvNY15bBr4/aj52PM3Cp5rjINCHuEmJlW1xt37csFDEoZXbO0ftr1/moOkxtZJoWIyeGQZStZXWVHdmRHXj6yHR3p5TK/d8CgbYrefmANOAg5ZFLCm6JsqEKWhXA1ySKOMo6y59i3MQgsED23hQMg9Larli9Pi2BMdwkME/s2g+cqY5apAqPn283Xht8kjCZltTlXbJ1svhcrytEjkMOCIoPM7wLtZLRXCfmC7FuUrOKgc8cZXdnzrNc+Mcdte+j5up/AMcvumMi0kW1H671KCAaimBg2AaNEfTIuBOW52BW23wWA2HvY3Eq6j0AB+8YkqzxbxUftYRUi1TfrybPsIRu2I5Are43ugzluWUJXA4KyIJktQ5z5f/Y59v/ZsCBbfj3ftrmMinF9jrnfc81D9c+4Z5E5lp0jumfWs21z+Njjto7PtX7pecqbZNcAfReDSCCHbdtx7Ksiz51LIcsQy7IZNT4Tc8yyvmwds/fNSnZOZdsrO3Z13P6W7R/9rhAwO48ssy/LOroZYgG/az1/R3bk5SRWz7J7rb5rnVY4mFgeL9X8LteqX+rccdfYNe1mrzkvlrwUym7ZnXbvGKdD7MiO7MjLT7ajI71c9KMdMGibIg9/1khaJGyy64QBoJwmV5Os8ZENzUgZfROTxIbgyBt+0pfrGX/stL+2ifPwp4SwqQEhsbRAIlunceWUx77JqPGijc+CG1UCq6ODe4vZKqNv7ykSWFdDXH6dC2Pao41jO23588QSmfL33sQxbY7jDNILBHbOXf45m/7+TxLC4SJCPhVrxFoR8KKcR2JuKZG02B9WWubaIY6hUiawO1rAA4S3dUnUppb10fP1VG4D5XmqMjomNn1d1J5WIbIAXjaZsEIeh7j2jQh9DAFAs+F0YkXUCSCS7qm5ofNrvmxiQljWBQTDXZ5aG3bUN/e2YX0WMFSYnkJ32gSQ82p5VlSmrIGeIwAaVwvlsZJl49j2FuNE4HCKy1u1zChgNu5+On/cvYe4sWTZXHlGx6Dq0Sa8UW+ZMDY6jN5/ksBQsqJ5kc0lYcskgKVAyEVkAR0BT9m8UGWC0aVcSQKJ87iwTs1t1cmyBsVu07qkOmldE9sRXwclMRc4OS7ZqsZNbP4vEFl9k2P07XdZBp7aKiWE8Sp82LaZyltlNETVAogV86wso/RGynaNqR2jZkdeTiLH2dUcC+DWEeW9y4aR3UoZB2jbveq5yidGdvaNWtmQZK3Lt7qu1yoWANLaeatCr8TutHuJgMXtguw7siM78tKU7ehILxf9aAcMugbJek+yCXDh6kDKuJCJq91b549T/McZYxCMZSWzXccZgeuMMoEsO0PPzXrHrGQBBXvMfs9lzpUxV8j8ptCmMiHUToml8WWdJAAaOXNPhSoJ9JLIcJfhrbCztvluf4PRtzvAlaEv4+pq2yfLFhnXPldjV4gZonNkOApwGUdjtkwXzPVZD6VlQGTLBKMMIFsHOz6zbJ+sV0vnWnaPBTayjKkse0XXZMumMDEds3XUNRZcfS4Zt/Bm2RVXu4ftl2zZs+32fGLDjyzzwyrzVmG1oV/ZOZZV1C07aNyctGIBFOv1zsrVNixb7yyjatyakS2P/Zutw7g5ZUM37D2vNh/1fdy4GlcXgUp2bI07z5Yzlzkv2352bkjsG3rGrfH2Gdk1xc75cUy/Gy07zKAd+XqQ55tX2wVFr1c0zwXi27cqqhx2jgnMsk6Y51sn9Aw9JzV/s0zU7J7zcpCUm1Pu52IL63fbn3a/z+5L43SiHdmRHXnpyw4z6OtMpMjLA20lC87YDcC+TWeC0VwhWUkyv03gWC4SvVEpT3hzUYoDTaYJb9oa4PIDreCMzy0Ck8cCCQqRUr1sziIZSVmPRrZsen4fmPG/bfr/lwhvNBOLYcMfrwGvAA7iwKAhgUnwNPBaU985gsKzigOKnmY0RKPq61nxbbxGeN15w7dPGzhDYApsEZJR9xn/liEI4RtqE40BsWMgeOxl+MWZazDn5X1bKU/KM/5+k769WqYNrejZAgtUVuWOkYJnE1xa4EG/6W1Q6iPlQbGGtgxTjVcxRhS6onxLEo1x1VFve9KYzvl7TfvnZhlUeX/uXbg3krVwfbPBlaDoOOPdhjkKSJGH1PapWCgK5bxaSI9y70z6cxNfnl7mfgpnzL5dz46VOoGlJNaNxrrYLFvmukO4MWHrad+qZZkjDX9/rQcCeisExpoFIMHNPT0/u55tMh4gstcnuPG7bH6z91H4pWWVQQgf07iSKORT9xZbUetCh5B8U+wYm6/Mrr02DFdzMWU0abdCaWu4cbrBleFytjw6rjxZkiEOaLfzpoNbG9VHMW4MibWZzTlUJoSS2jlkn6G5a9e7myXbNah2jJkd2ZHrlzxu/ZjD5Zc74L+L/WvzvHVx6+uS/6wz+obH51rHK4Q34GqPlL5iWbjj8p/dKJHzEF5YYm45+G7kWiT2snV6Xc2RWPQf5ZOTLmPzQ1pW/M3MZ7QjO7IjL0y2oyO9XPSj53O074iXcd5dq8Rn44Tt8efLfWHvk0XnZFRf4XmOwmuUxcDp4RQFbT5Z+i9c6WGym/44z4W83mLryADP4xSLMgEEkDEmz1bR/F7CKTzzuJCxvbikttM4I7OCM6Dq/jOHC3GZYvSV5JalYMM5ZMzrIwWpw+hrzu09EvP/52J02TbMGqIy7rP9nhUpCDKMpRwI6LNMguy4sgCBnjdO8cl6EMexc67GcIm4cgzrzWuxOaa36ZVNmYfmPCs2d8tzLYhiiFkDHMYrWdn/27Ace04WNLLXWuZF9p4qr8qssT+uX7Jin21/t4mRMefI6I8IObV0XXYNSTPX2eMWzLX3tzLuNwsSj6uHLUtKGLtW7NzI5v1R+9l8Pdk6wCggbgE+G1IqsMje+2qe4HEMIssGtIDgOMmukVrXVIfsGxXtHIGwZmt8jnMa6PznMrrGMaduhiTX8dmOfOITn+A7vuM72L9/P1EU8cd//MfPe83HPvYxXv/611Mqlbjjjjv44Ac/uM2n7siOvHiS3S+ze6fVk6zuNO6cIkH/mcO9FfYAzml2EAcQ7cbpTnM4fWmKAGwXM/cfJ3rWuM+49flmGTDZ9dAe3+66dyPLnGVuZfvV9nchc15hzLn2c7PX9x3ZkR15YfK1qB/tMIOeQ7Kbi97CIJaEze2hxV5hS7peTBQIrIXU/F8hQjJCqzimiBg/YpVkmRGVEvQ91WPal0We/xSnKIj1Y/Ph2IFp32JjgSEZr8pNMoUDZgY4BoO85a/EsZFy/q9AH9VnAgfsTOPYH/cC9+CAoAru7VNtX+dpX8+7cUygg4Q3ny3554v5EOGUI3nC1CZ689OMv++yb0d51cR40UYtj0yNkGTXJvdV31ij2Ob/EGCg38uZtpZRiK9bCcfCyPl2WSe8fnqDwFxRjhL1wQaBqSR2g5gTWQBonIhWLsXL5ozRGK4QWER7/PEDvq5VgjF+L05pncT13wqu77WQqIzyjGkxzObB0ZiOCcyUxNTb0qo1n2qMsnRSwtxq+nsod1C2/rrfkNB/ag/lcoIAeMz6eiU4JXyLUWq+8uOME9U7C9q1CEwbPX+SENKpN+QlhLeaCVhV+TRn1wnApzUypJCqnJbFZsM2Baxk17iif3aL8MYv1TVhNJG3PMoJo29G0z1VfjF7xCzLJtdXGS0oKuZO3ZRZbK2sKAxPjC2NtUnc+MQ8r4FbHzSfJGrvcfdPcetNz99P7D57D5urS2B0hGs7Ox7FUhOzK/tMC3DbvEI3U240M6jZbPKa17yGH/7hH+Z973vf855/8uRJvu3bvo0f+7Ef4z/9p//ERz7yEX7kR36Effv28eCDD27z6TuyI9cvVn+wep79XQBAiaAz2nVP4HEBtx5M4XS+/cBhnO4zR8jnuE7QTQq4fXCWsB5v+fMajA83F4hRZHRftvsGjA8jv9EifQfC/mwda1dbk2+k2PA5CGCdwDcxgPXCFpu3sELI9WaZo6qP6ijwSHK1/IY7siM78tKTG8kMulX60Q4YtA2xDIRxngwZcnbzsoar9RzLC14w58tTY19DrvOyG3w+B7G/UFRU3VeGijVO7P0keq7qZRUDC1TJa9VgNBxiP+G17lIu1AZWaSoSAIQZ/7cUQZwGI1UhQzWcIWiNZhlfNpa+TABq9Gxt0lOEEI2BOQ5BUbP9JwN3XOiJnqG6CbzBlFHKg8pH5hqb/DYhKIo6R3TtIcGLZMeIXkFt+8uyCsYpb9k6SnHMm+/23InIg5mRUzS3UpiLoTZ0/RVH0IlgbwTTKcxE0B46RSdv7mfZDOPAB/ubxpENbbOgZ5b5JZBD/7eMDctCykqWUZWdS/GY71JQBWJoLAm8eK7wHc2HrNhE2PZcCO1jWViq8zhWj2WnZD3TNpnoOJaOZeJkxTKh8oQkyIm5RxYMUj9kwwuGuLbTM9W2tlzqE31XX2VDHsYxorJtYu8PwRCz5wwy5+p+NjR2nNj8Zmo/fbIG1NXYWmpTlU3z42rszOdilN5I2a43S+dubm6OHC+VSpRKpSvOf+9738t73/vea77/r//6r3P06FH+zb/5NwDce++9fOpTn+IXfuEXdsCgr2Gx+ohlXWbn9PXcd9z/s/rcuGNyCClEG65cz+T4EShg176++b8FGcSKnvT/LzPqTBLwb9mNYuhqrbSOSCtZVpD2GK1TWYfCtcq49hr3+9X6Sfpv0RyzzCkIOQlvhGTHl3Wm2DW8TOgjOc0S3N7YIThxdF6FsPdof5Rjxra1ZR9fbT/ekR3ZkZeebEdHernoRztg0DZERi1c6aW2m1YW5Z8kvC0nJnjcpwle392EBNBb5lp5xLMGbL8H+WEoC4SEyipPijMaa4zfmAUYTBE8NNqEj/jyThLeVFUm5PQoArcBByJoxfBXw+AtkcEso7mEY/jUcYBSzhfkaRzrZwr3NrYl3x5L/vzEH5/x3+sEAEpv1trjz5knKGWvBT7py1H358sQLgGv8r+rTax3X3lP9hFyHbX9cycIuXtinPduy7ejzr3dt/2ML98mjmFzEjhLAPvmccyalNG8TVP+HDGZVDYBEVqErAFp+x2gEMG3VOBUy40nsSMO+HMP49hZAk8O5mB6BvJ9KE26+m9swu2H4JHjsNmDY/Put11lyCdQKcCfPgWzPddnMfAoQckRm+UALmdTFswRayP1badcRqpHmVHFV8pXhfDmM7VDiVHAR8CEQAzLvLPGRNYQJ9P2aleFGsqTq1w2Yjfpr0ACsX0EiAqA0voRm/M3CfNU7RMRctzUCW/UuppIkdVYVf0VmrlMyJklxqBlvIkBqHaS0iqjZcOUUf0I4a16EFhPWaMtMtdqLA8JeZnEyFE9thgF/DZN28n7WiK8+U5tbjcyu94qd9mQME67jILNKq+AXvWDypQHJgvQNRuAFPxJRo0weY7LuDHQN/dI/HEBqMrjoflpk4jDrTMSrpcZdOjQoZHjP/MzP8PP/uzPvuDyfPazn+U973nPyLEHH3yQn/zJn3zB996Rl6Zo757CzVs5vPRyiA6BbfN8Y1XzSDqOZfVYUKZkfi9l/q89TeCFAIOeL48+do0UO0RMRbF2EnOtBWrEIlkmrIsbOKb0FqNvubQOFrXTpL/fGmEfVf3KhJd3iKkpEEP30np6LUaO3h6p9c7u+1nQyYbva09R+5QJepcYTtbpoPyb2dDc6xW1RwE3rioEEGeG4KzU/qA9p+brK5Cu58vVIOSNrPqPGKMNXP8t4db4TcKbW3VvmwPyZuVp2pEd2ZEXJtfDDHqp60c7YNA2xA6ALDhjvUPZzVRKgRSQij93ArfJ5HAG+rL/aNO3rIrsRpEMgtdIyWytF1rnW9aMFXlllLxQyoLOO+qv3Qec8t/nCK8TF725BqzHEA+D0SuFwibOk0G7RTA2n8YZTNOEUKlJnDKz7p+zAtxhnrkLpxxJCZzAbbYCmmJfZoE/ZcJbyGTA72bUgyaDXu1UwjFfOmkw3GdxioJlQe3zZdEb0Aq+/AMc2NLAJdw+igs5WieEqNQIr962fSsmhRSsceF92YUo6w3MA/dUcrS7KWma0EohTWE+cs+8K4Jjpt735mG2BrkOTNXdPVa7cHQXVE/DuT7cX4XpaahMurGXL8EXjztgIfvabI31HgHcglEvnAVwpAgpzE510Fiy7Bn1k9ojm6cmy5KxCm4WALBggmV4yItnWVt2jlnvZWL+qiwy6i0jTJJlL2muy2OoMmh9KZljz7X5WCaWxoPAEHz5ywTAw4I2Air0f61lem6X0VDVLGNL96+Y3yyTR9/t/wX8yRCw7TEgJL63IKedn5rrWbacxpbAIkv5F1g1ZDyDTJthFoSJgGIO0mT0eEIAm3RcTC61lw3PtG0tNlU583zLktOzb3aYxPUyg86ePUu9Xr98fJzX63pkYWGBPXv2jBzbs2cPm5ubtNttKpXKi/KcHXnpiJxm07g9RCzlLcIaY9kWz3UffQTmZwGMKk6PsHqQDHuBJzb/iw0V0osfNvxHCfwFeGjtETiSMrp/93H732UnGSGsqOXvuUYIAZNzzZZHgEbkz1dIs3WOZHPZqAxymoxjFI0T7RsKVxMLRvXVxzKc7R6q/VbMKTEu7csltKeqjArzfzFE5dQbbetALXJ62V6cTjdLWJejNNRVYWIQHH0bBDBI7SEAULqhXnChPd5GC6j/t8PI2pEd2ZFbK9fDDHqp60c7YNA1it4qlFXUBRbIGBBIkMMxVmTMTOI2hv04Vkgfx74p4xSeeRzw8VX/25045sglghE7STDgq2nYQC17RIaoyirQ5jW+jLt8OaYKHowYwr6KM3baW5CLoZ3C7hTW0wCyrPh7niOAGKeA+TQAQTKAlLsI3w7ruI1Txpgmkjw+YohIcbDJY2WUKxeQ3jaFb1sxc1TON0ZQz0O1HzbugW87GaVPMRoGh2//GvDmChSmI15Zz/HsMwN2TUKzAwfqEHVcmRq+gjkceGdDT6q+fSf9b2IwyRvXxCl3HRyDqeTb9IxvzzUCI0XllcGZNXwtSCAltQUcLBT4lh/7e7y9dZbW+YdY/eoCrXU4egxyKcxVIqaqMww2uwyWm+yegVIOeitQrkBnC6qTEJdgogbDFsQ5iGKIIthchUYXnu7BwQjecggGPaguwFdMXQvA6+vw1BaspKG9Y0IOnNS3uxR1Me4sqGBzI8lzCOGtaDYsS+0gcKZCYCkJGKnixqRyzIAzBJq+3ZUXSjlvGgRvoA0nUt4p9YcVKZ0waqzIKBEgIkaNyql20/i3nmR7D7GAML+LVWNzD1kwRTkrxGgR48cCDhZM6pvvYivh/5Yy9dJ8smC5FF/Vwybe1Bv9rJfUhnWpzwUAad4LYJZRIyByiOtDMbEKBNbVLG4dFVAjw0qisSUjrGqu1e+DPvTTMIa6vg3tW95U7yJuPRKQl5j6KA+S/q8wAo0frd9TBEZB9m1kN1pU5u2cD1Cv10eUnR35+hUB4VpP7HquY3bNtPqC1kM5YMRASQnh5mv+Iyan1goxTuzLGiQlAsOjhptjc7i9v16CShGKeSgX3acApD3ot6DbhH4y6jAQS0l5yuQYsW8VTHDrspghFrzSmtr0vy0TnEQQwpC6XBlepo99yUOZkNvRhqfJWWWdMTpfwI0FubUuZfetGk5/3O0/cwRHJ4Q9wIJnAnPa/qMcQJalNcT1pXUkCnDr+e8rhLx1EjleLPvIOoEs6GYdCYeANwD3TMKc133KEUxXYLoGpSLketDdgOYGDDqQT6EY+T4dQNyFXj84cwTSCSSUTlj17QQhF6Rl3irHZBs3Tq6VbbAjX5siZ6d1nu3IS0+2oyO9XPSjHTDoGsWyFiTaYCzin8Mt/gUCXTclgAQHcGBRggN8SoSEynM4b8IF3CvYRROW10dJTOXdsiFhUjwUxqKFRBTXe/05d+CMkT2FYPQeKkOhAEtNBwatJz60y3tFpny5+gSFZct/mkA9CYpH9jXcUpJWx7SpFBmFfUEwsuVNUz26vt1s7g55VeSViXCsl2oeyn13zyYhRGTg/3+eYJRLcaj79n9PCdLZmNfvybP/+IBjNegMYXYKWv7k5YZ7/iYh7GbdtLfYP31CXhUBel2CMXqXf66o0Kd8m1YYzYekdrKKjcQyEjRG53N53vBt30K9+wg8eoLN9gLr5+HQqyBKIK1HMDtBfyGmc6JJeR8kDVePOIZOE0pViPIOHBrigCAiBwa1GrC4CueH8JocvHseOm24tAAXCcDpBHCsCrsbUEgDOKdxYseBpdWrXpYZp7qPhMMRPKo2pMxuqKL1i1UkNtkGo4ufkpMrvEqGuJJLx4RQNs01KdiWlQLBMFE/2U1Da4aUceXcsiGa1oga54HIgkFSegWUaT2S51wigyLx9V33x7PsE9VDbaFjAsessWHLZMOrhpm/ms8Kv9BYbZj7j7teAI5yiYlxqPEi8NDOE/VN25dL7EGV0Yb0WZZPnyvHodp2OAzjCV929Z81OtT2FUZzaXXM75Pm/9ZLDAGgrBHAu5udXNTW51rPv5Gyd+9eFhcXR44tLi5Sr9d3WEEvUdH+atc768CwDEu7/mgfs6FPeh261nmtSWKbQGDaCfCRvmXZg8pdOIUDTfbiEzZHUC+4va5Qcntf2b81oduCzQQ2WsFBMCQANBCYHx0C4CEZ4vQE6UvPNVcE9KieAgqyYsPYyowyVxR+q31T7a/2htH2t3ulwtgsKGN1jqpvs7twjkzpuTpX+5jC61SPjq97g9E8jlrbVc8Nwp6mPlcbCzQb1xbamy3QqH7PgoxlXGqBNwJvrcDeXZArOL2nWofKPJeVgc2LsBpDawuSoXOIpalzfA0HUOgHB0jelMWmjNCY035WIIRMq83kgLJO1B35+hTNC9gBg17Ksh0d6eWiH+2AQdcor8JN1E3cpqSNTIv7BG4TKAP3ETa1RYLxn+I8HAIItOEJBV4iABN3RrCahtCpaeDNMzBZh3Iezi/B6S7M91yOmIUEDkdwIoV7CrBrFxTLUIkdU+ZYGYZ9D1S0oF6DQQPWVh0VdogLi2oN4XzqNqpTvq5iKOHrUiQoQ5sE40lAjcI7tPnK4MPXIyGwQVLfRlX/fZrwxh3FV+/CMWc2/HlvAGYnYKkNW8Pw5rVvjuHuIiz13TV7gM/j9vZXAH8NvBnPFIhgfwEu9OGd9xcYHB9w8J49FPbcxr7KJWpJi3p5gYkpKOSg14NmCzZiaKUO9BBToYED8JZxgFUD522UclTxY+Jx356TBLaK2CgzhLeEJTjg73374ZFN+HIjvGnk84TcRI/hGF8XcYmd33Vfif96vMsPvOMYpVoBasfgwBsoHrzI5GAdagVoJSRrCenWBpsXezzzVYjPwmIPtrZgGMG5JtQSmOrCwho8lMKlVSi2YL4IpzbgXM+1+8XUMaXa6y4J9b1pyNmyCyh1YDIPu3phc7PKkgV72oR5dJefDxdxb6A7SQD2LKtLXlQtZDIW5FHUeBTzRuwbGGX0tRl9q1iOwGoRfV9eziFBAbTllwKqnBJWSdUzbbiVniXlWcCxyiSPIQSQQOURNV311jUKs5LBtEVgm8jjK6aKAOWuuY9l88jzqtw+anvVSXVVCBqEvhVwovPV/sqn1SesETq3w6hHVeCQjArLOJMBYUHDJqOgkAxS1csyO7Ngs0CuxN/H9kPfH9vy59qE7mqLpimL7SeNVTGNcoR8V8oTJRGjSWygScIb2m6mYvhSA4Pe+ta38ud//ucjx/7H//gfvPWtb73BT96R7cgkDiCYILA1tB4JuLFMRwsQCYRQiI32xnXC+qiPwPDdOH1BbGkxcvVR3jk9u4h/c1cB9szArkmYK0ClD4M2tJuw2YG0BVHBh6X1YbUNq4mbtwJ7uqZ8awS9UPuDQAmB9PYNns8nV5t/WlO0Zgz8fTcZzV+3B6cvKPxNe6RlqyoXXoewF9QIYJdCWGeA3SXYvR9mZ2EmD3MDmOpAvuNZUx2Xw5IE4hTixDmdGEIvDWu43X+zjDEBagLSlE9IeqIAL+nSYpJqrZc+GpvzNb4EEs4Dd+fhLffCK/dNsDdtkdtKWFmE5sAxoYvnIS041s/6Fqw2oNODNIF87HTlZAjdQXiDmxxU2mtsvkfpAXmCw9IyvQSmady0GWVlxeaTy/zffjSH1IY324GwIy+OSK/aDjN3R26+vJTAoBdLP9oBg65RXonbTC7ijFTFBldwC/88bgGv4Qz/S4QktA8TjJMVfz8lw5VBuY7LK3MIp8gci+Hi0D1vDueJeXAGbjsIpTL8RReSTbhzAKUYvpDAuyP4sxT+dgFecQBqdcgX3CZXmnZerhTor0BhAjYXIG07kGgQuTItJXAap9g8jWPRLBOMEiva0K0yFhEo3fLO2fOnCQCPFINV326pP17z5y/5dpsHnsUpPbuB1wOvrcFGH04OHVD0BPA3IpgqwcWO3/iBT/t2vjeCz6TOIzQNPBvDAyV4ZgA/9foCjy0PufNNe/jc4bdwpPMIlfMLTJYXmKhDIQ8ry9BqwsWCK+dTvs/EkrqIC/FL/f9FGVZs/BGCEWzBoGUCCFYjKMu7gR/ZB/+/JIBBb/NjqQS82pfhtbjN/54Y/h/3lfjs2R5/923HKNYKUDwG+++ndPCT5NobUC1Ad8BwfUja3mDrPDzxpCvTY759O74ee4CJJdc3XwHOrLp+O+z74jTuFbgLOKZQY9WBQfekri7LwKEIil2YzLn+ECiYzbclg7rl+7+HY81F/j6vIrzSW8CFwIMKYfxBSCjdwSmTGrMVAgAgY0RKu0AXgXFwJRika7LMJClhmPNEkY8ZNQLkiRUokQWDrEc5NecIeLLhBwKDLNtKAIvqBa7NpeRDaOssGCQwVveRki5AR+daYEwAjUBw0ePVHsonoRxAMg5rBNZkjcDG2TT1V74OhehZ4KlAAIotGNQy54kdpPU1C8wJmJHSpXsKDLL3FRi0QQD/pcTLMLHAjtpPRoD1zcQE402hixb805rYwXnhBZDdTDBou8rodhXXRqPB8ePHL///5MmTPPLII8zOznL48GF++qd/mvPnz/M7v/M7APzYj/0Yv/zLv8w/+kf/iB/+4R/mox/9KP/lv/wXPvShD23zyTtyoyTC7Wu34fYu7YMbuLmjEB8ZupqD0h+0DgngbRAcTVoflOdlCrc+1Al7QRkHfEzjnD1ThMT4ClEq+N/2F+HgHpjdB6WKY8JeOgsbK7DSDQa5GM2XcLqI6iKwyoaGad3VXLDAi2VqP18b2vVp3O9yQkaEtVbgCrh9VqFve3H7aXZt7jAKYqndtW4XfFvuxjmdXlmCe26DqTu4nAE/3YDWGmwsu1CqdtOxZZKh+wz6LqyONCTMFxikPs4CGX3CPqW9V32uzzJunZfzMU8IQxP4Iua8GMcCko4C31yAb3ollF43AYsdVj+fsHoJFppBt0gIqRHWCaFgApgggPkaC6qTxrgAsAkCMFkjOJAFBklXzrLcLKNJddBH+5ruZRlYYhdZNvWO3DixDOIXS6Sb7shLV7ajI71c9KMdMOh5JEd43ecUYeMXYKG4Z3nUO3jGBM5QXiFsEl3Cm5Os932FkCxQxuxTKRwnGE17gXQIxXqePAPKRbh9Lsd9d0bka2XirQq3H9zPu559nEOvfC21XT2Kw0Xi3iZRv82gDavnYKvrwpySEqxsujdObQ6hNHCMoEsEuu4F/12GTpY2LOXFKify3FsFRZ66IeEtWxWCUSQDC0JeFqtkQQjjuQQMY5iYgom9UCqU2NPpcijKcWxuD4MTF5jagne9fj+7Fy7x7ksDZl6xn10rHb4nbvPGVz/Axc99nHe9883sOrVEjQal17yFqS/8OdXdU0ytnaR37jzHT27ylQ48uQCTfTjehEoKy0NX93XfRirffl/n3QQv3SQhzC7CAX1zOIVUyqbaUwBjG6fofsuhMru/43u4648+yrevLvLGb3s7+7dyPPi5T3Dvu95A/ssLfF9ti3e88htJ/ueH+IbveC+11+3lzU98iOi17+Gzf/oxpptd8uunWHl6i9VLKeWlAevtIQu+U5c34IQv2zmCEbzqyzfvvye+fFIopdh3gGEKf9GEdgLN1IGHU75em8DiIOQDsGPeKs0QFKRp/9yj/v4rOGDqNgIzY5LR5OSiw1uK/JAADsirJmW3ZZ6t3wT0SDHTZixjXyCOxrTmRIWQBwF/vfVeWiXBhq4NzW9DQvLlbFiQFEa1k56v+TgkeLxb5lrLuLH3iwiglxZ/KZ/ZflFbSJHV2iUPpcALMYg2zPVZME2AiZ6VN9cpdGCCAJCVCSyAIaMharMEJpc1tOzYskZVJ3OODcez5dX9rFGmdkwYVdC0ntt2gAACam1oE/oMAuAkYMoCdzKKVU4bNncz5UYzg774xS/yrne96/L/P/CBDwDwAz/wA3zwgx/k4sWLnDlz5vLvR48e5UMf+hA/9VM/xS/90i9x8OBBfvM3f3PntfIvESnh5u4UwcnRw62DTQIzRvlvsjqEgAgd17zQuipWn1hGYkAohKzAKMNI67uAdBnaWksGAxf6gz++2oOza3Bm6PSLJiEP0BZu/5JOJNaFZWGoXFY/gjB3t2sMXA0QEmCsdciuaR0CU8VHuFHH5TqcnYOoBMMcJFGom/plEEEaQ5KHtFwjN7GX0sQ+JqtzzOcr7Cl2KU5fgOHj9BYarJ1zbxvdaMFWAzodGHhEKU0gSVwuyl4aALMW4W1r2hcUSqbfBNpZsEiMLtW7hlv/54ADZZjbBVHRM5AS5/wslCBfjsiXqsSlOrnyFKXiJHO5Ikdokpt7lq2nW5w6mfDURXiy7/pde1tK6HvpEAIktb5b4KdnyiwjUXuH1RekM2m8qA/tOLJjKdv3GvsWGMI8U7/vAAk3T6yd82KI1Qt35KUrN5IZdKv0ox0w6HmkgDPwJ3FGqgyLOgEI0UIuQ3ARZ2SfJIR+yTiVsQoOLNJGI4NjAxf684UEHiEABvcAaR8qsyVoD6mVU/bN5HnLW/Owa5a7Fncx9c63c++fPkPh+x4k6m3C+S/ApROw3Kb1JCw87ZSeJ1N33wupY4Gc9nWUB6xOyOehsip8x9JXBQDJsFP4hAwoKWizOGO+7b9LYVkgsIfk7ZGX5TyjHo5JX84zODBoZh5m74Nje2qw3CWNi0T3HuPs71xg1wJ8xzcfo//5DWpbA17/TbfzF4+s8c2lS7ziH/wd/ujCZ/lb/6/v4dzvfYla/Szld/4Q83/0USYOzbHrE1+m/fAJPvEEfGbgvIb3AJ/FsXG6Q1e/ZV/XJg7gugOXD0qsDYXDxATW2N0EL+ZXCGAYuNfOTxIYAT94zwQH/58/zf0PPctdp5Z4x/vfx+LZEvUnP8WDf/9B/vSXvsD37TvNgb//fkqPfYT/2z/5MVaLd/Otn3gSvvHv8Kf/25u4+8QJKsBjacLTKdTp8QyOXRThwJsygd3WJ7A2Fvz/V3zZL/i/Z329Ul/3QQq/s+HqvAs35qdwAE6UQtwPc0RAnwUdZESX/P12AW/BhfUNfVkO+zFzwrfnnD9uw3Ji3Lg9RVCMBCzg6ygP34a5Vt7Krjnfsn8EXNpwIwi5qiYIb76TCMDNigDjCUaNIrW7wpbstQkhBxOEOSUmy8BfuxsH3KkOAljzjOblAQe4nWFUibHKrNa1vr/fNK5tzxHAapVVc7+CW/fUr2IypQTGoLzz+l3hHlKqZ3HzCtMemk993Ljr4MbWEiFvkkAZy1RUPeTltaFWUtIFqEEIi8gaWnrGIHMPUfoF2CiMToZLA7emKXQNAuNB4wCCB1nfdVzrqEDFryX5xm/8RtL06irSBz/4wbHXPPzwwzewVDtyPRIRcsnMERw5DQIY1Dafaw1f0Vyza2FEyFMo0KBAANLFeJQRLyed1uzLYPsAGuuO1bLcdHqFnG8XCG88lV4nEAOuXbG/ntBOyzK4GhhkWVUDc9yGEIm1PhfBHfNw5BVQnCXQSPwmcvn+opqUI5ifhoOvgUNvg92vIqrOE3U34MzH4fMXaX+5wckvwakttwY305AnzToJLLghcNCGCcYER2qX0bxKFiApm/Pk9DiEYwu/YQLuvgtq04GVVKjhNoq5mGh6GqaPwuwxmDoMpTrR1gWiL62w+n+d5ZNfgY8OnQ68SqgHBKaSHCplQn4m+9EeY/dP6RUVwpi04b/dzLM0NwSK2X7PjiOBUsrNpD1Yzgm18Q6gcPPkxWxrq7vsyNen3Cr9aAcMMnIPboM7gFNkdhMM1PP+2AIhcbA2Cuv16uMMmvMEQ0CbfBGnOPVxBqG8IqK6WobBGiG0pezL9ak25E4PaHXhq5tw5zBh48yAfLNFNFiDlWcZbA4pLjxN63yD9VOrbK132diElQV4tOuSQ5/BbT7r/r5N/zwl0JWBbUEeGNUlpHS0fF3a5vccIV5dhqIUhFXCZigAQqEQeQK7weYe0YYrT/7DCexbgTtPwoFhj3QVJvcN6V5YZtiHo3cWKO/dTeHINAcqx6i96u0cPveXzN/5ABQqHNg7Sa5YIt1aJJdfpvnQp3l2sU/7i+d5+ESDrTV4IgnlO+fLLu9RGQeSqE4KuZHBqNxSq4Qwmr5v71kCxRyCYd0geBuLQJwOYekx4vY6s0dq5HffTnV9iQNvvJf8/lcRTz7E9KsfICpNsOvwLuL6XvJnH2e61GFw9klObW2yPkwo4gCcC4Q3nAkUEQtFHjhrmIqFYcE9S/WWZw9/fNL/PUgADtu4uaO6CiysEpIX4+veIYSFrfk2X/JteMr/Xx5egQeYvwplmiGEEFkZmo9l2VhGj613ymjy0UHmGrWfxqn15NnvAnpTc531/OXNNQIWsuwWCx5YZovmnejhAn865h4Fc52OyWOeMNoWEv2mcinPjT1PdVaIQTYnhn1elumkOolFoL7S+qcxJmNQzxwQwtmy4QRi9lgjMfts1U0hHLad1a7qI52nOqlcahuN/RKBnWTHUzZ0wD7f3tt+t+fZ/r7Zkh3P13L+jnzti5h6CsWRcay9fogD2sVSEVArIOB6x0lsPlovtb8ojHRAYG5o7bHhNNJDBrjwpbUe9IbuLZfaH5cIepD2iptpXGfXiec6J80cE3jVxOkm54A9KexqQuUiTA2hNAHFKsQTEdTnYPYOqO+HfD7EHE1Mw8QeR7HZeJb04qO0z22x/MwTrDzd4sICPNtzjsQNAlhn21rOmJ75CFjR2qL1Wk6LTQIT3BrDNsSsgOubKr7fYyjuqpC78xXEE4dIKJArQTQBTMQOJart9hTySVg/Tf/Ucc490ebLq/DUMLDfpTNoP1T/W2BOOqrY7Bprdm/Xei4gUvuX9gg5ClNTL8uIez5R21omnGXxvpB5tiMvDXkha02Wjb4jN0a2oyO9XObjDhhk5NuAT/m/J4B3xvBI6t6E9AmcAbvuz1XYSJVAg7Xx2xos2jzAKUwHcZvPMdxGcMH/rs1Qm8pZnAe87P8+DTyyDkuf7nIudQkNvzPqc7AF9ckeM0dWSZ68QPNCl8rDf87Kx3s89sSQp4YJT6bwbAKfHY6+NQnChqdnq8xdwquaV/05ZYLhleCSJS/6vxcIlO0iDvTYRQg32iAkoxZYtM4oU6qKU2TUtlZ5ED22A/xeAqeegfc8C995X4PhJtw512Pzi0/RacHbvrVC/p67SYvneeUd3058/w/xpqc/TeVv/31alwa8+r49kK8wvPAYuf4SS7/zi3zsiQ6HTj7E73cSnkpgOXFMnhjH0Cr7PkhwwMdFQiLxGAcSFvyxeZy3UWyVqq/nPsJbglYJ4XMlHAgSEZTrNOnC479PtHqWg2/ZT3z0bUyv/gVT3//tRK/4bpj/P9n9PX+frV6d2990D8zfTekvfol9E6t0P/cnfGF1iQu+XdWGVYKSJiaF2lkGedn/VZ4ZGb0CvPL+r9gsAnfu8WPg1b4Of41TrO/0f8EpcAXfPpuERVIhaAWcd3afb5+LuHlYxDGOOr59FSIWM8qEqeFyMz3OlSCAPJWKwdf4z4ZzSdHUPJBypfAhzW2xiQSkqZ1hFIjyKRZGWHUWACkzGo4lQM4Cw1IqYTQJ525c/qYt3PgT02bDlNcu8DKaNgleZnks9ZwszVxG1vnM/WQoqe7rjG56Ald6mfM0p6Xcq6yXmWaExKAbvqx10277/D205mq9zfvzFOIxQQDIrMiLrrXMAn82bEztYcHAIsFrrblUw7W/fheIvkkAhi3gpHYXg0gGhtpGZVS5bRluloxjJTzf+TvytS8TuLE6gdNJ9hBy/W3i1qAFwr5+LeDG84kMa4UUab9VyL3C0eRU0Vot54W9toxjOU6mUGi59WMFp8PYvEB2/buZY/taDAytH1bkWBPzZB1XjxYwvAT9VTi6H+b2w8w+iGciOHoXvP5H4K5vg0LVFKIPq8fhwhfhxP+ERz7K2sfa/PVXhnyuP+DxoctlqbVaa6+YKnYtG5iP9FqB25YZJAbwCmFNjwmJ9q3ImXoAF/ZWPDYN3/aDRPd+L7nCRHhIhPsniiGKoL0Kn/tFOp/4n3z6v3X4UNvpddZp+1yicF/pTRqLOXOODT8Wa0djNSU4iwUEVQg5/651zbUOyHGOgp21+OtXNP6+HsaAZVHeCtmOjvRy6Y8dMMjIwRrs68DBSWitw2wJcp3w1ohNAstBYRQpIZbcKv1KYlfFbQqHcMbb/px7A9bByIXYaIMpE54jY6NGeDVlBefFOjlwyksb5535QhNqacru5ZTyqR6razB1usvi2oDjHTiTOgBG10iZUCw2hMGqkI2IkNRWYUQlc76ovdrQlQRV10oR1HOaBOWkS3ittJ0klgINo3k7tPFqAegAiwk8ncDnN2HYhNZFWFlO6LXg4OqQ5mNnmdhqMHnbOsNLz8DmJvH6GRafWKWw2KL90FN8ZaVNbZiwtNzh+BA2O0MWBiGporyObUbZMzJyLfgnhpdYPwJPmr7MChNc9scq/n5iD6muif/98bUhjUcvcmFzwNRqh9knP8fql7/KzK4mw7OPUFhdIdq6QOOZp4k312h8+VN84fGzFC+26D9+gs4wHYnNV99L+dLYVVvrrwACGcrK/aKyyRgeAsXY0cSbachDsIEbr1LABQw2cEaExoBCYOQZxNx/hcCk6uP6Q6CN5p3GmijSAhPUllbEMoErvbzZDVSAnPX02ZAn/V/9PjTXFRkd1za/gFWCLStIBsAGo7R/Wx5bZgEpdv4KHBIzyJYTAttJ9xvn0YTAblG55I1Xeyunjr3GhmGonQSsDM01WitUP5t3QXUTMCKjLsvK0rGWuY9EISAqj/rR1tu2YfbYOOVafaX1R/W2bZBdw9SmArUssykrGkMKrxCwqf+L6ZAFtG607DCDdmScKFymitu/xBQS0G5Dwey8vppYxg8EXUggTpz56HcxfWSIC3iQU0N7s0D1DiGPl9gZeQI7WmFlVofLil3DtwuW3gyx7BVwe+EycC6F2QEkLVhfh9mce8NsqbBFpXiCyuAh8rUypNDvQmuzT3P1DK2lp2mfv0j32Sbnl3o82nJvqr2IA27k0NB+obAp9Yn2csuwsXqG+lmguBwmliUzbh0Si+g88HQXZs72qX/5FNXGQxRr1cuviU9S3ItReu7NX+3OOq3HTrN8ocuJTspyEvRUjS3bjuP6V2WykmUE6Zj0Jrt3CQyKzDUWWMqb68bJuP1mR3bk61Gs/ngr9I8dZtDXuLz9ECTn4IH7YOLzcGgW+guOTnqS8cm9ZGRrU5SRtgfnvZjHbcrfARyJYa4MnTYUYrg0cEANBKbJF3Cb3W24kBcBMQ0c2+QvCV7oGvCb65DbgAOXgMcT1vqQnhzQ7btXxQ9wCtwGweCY8uWcwilCMs53EzZ4hZjPEowSGUR1nCdNzBfl92j78i4QXou85et/iaBolQjsBAFQii3PEbyNNpmszQvQx8V5nwT+7KwD1d7xV7A2hIMJHPpUi8f//A951d0DXrW3Tu+pJ2k/9iSVz/x7PvEfnmX/+iZn//zX+ffrDRoxrCTeszYI4R1D396iYEvJFEhklc/I11PjYJWQ7HDBH5/0bbDiP9+Sg6cSeDYdNRYH/rn/7rEu8bOPcE+rD80F9vziD/PQX3R5299/E/HZ/zf1L34JHv7PnPmNT7K3sMypZ/8uP/HxTd7WH3D2Ex+iPXTtlsPlczhJePVuBwcOQlBGpDx3/TngFOQKwXMmCreUkokynBvAyV5gia3gkj/XzWe/GS8n/XjQ+JggvDJeStCjuLEpUO0igWVRM+WRcSIA5ixuDFtGTUrIiRQR+lei8D4dUx4gKagQQCz1z5Qve55RA36OkFsJAnNK7BsBRk0Co0p1eRo3hsRaUnmKhLEFAUBT38S+LTUGLbCoELVpwpv5dL7YT1JcZTgp5G+doKxvEOb9gmlb1U1+ZYU7Cjju4dawS4S3uwjcEs3e9kXXl7VEyDcB4dX2CindMNcICBMrz4aK1AngvdZmPceCy/j61s3/VTcxJMXa0rgQqN0351tQq4UbJ4sExcUysLTGiGmnhPoKZ4l8W4iBdDNlhxm0I+OkitMHpv1H82MDN84vciVD8GoisFNOM60nJdy8mfLPk86STbyr9VROJO0BNjzJAgpy4gjI0npnc7UoWfC4sgqghdG96qUktlyqWwM4lcKeFZhZg9kTsCtO2F94ktvmT3Pkvt9gal8EKWycgxNfTXmyOeSZYZ/Twy4XBj0WerCUBr1nHGBmwTLJ860L2g/FRLf5oa62BvVwaQ42gKV1ePL/WuU1f/YbHKv/R+b2RkzOugTSg557y9naJVhch5NpwjP9FifaCWeGTk9pEnL7QBgrcnaNE7vGy9mWFTmBBERGhDA+7d/aE1T/GoGNPW4c2lyLLxfjckdurny9jAsLnipK4GbLDjPoa1z6A0gT6PTc2wmWh86wX2c07MR69y0LQEZQjfAKVOVPmY9hOgqGmzYdy7bQRqsBLo+wYu9lSMowbgAbiStDeQi9njc0+4FuKyVIf2HUYNIAEBNJdGMZWfLyCRSJCDl/5OmQwWVDRwbmu2VySJFQuIiMQtWtzCgzQWFLMJrPRW3XGbpjCx3XT2Xg+FbKubUmtZWY2uk1NhptNtZ67Dq1zNnldYadhOONLRYT34YE5RBGw4FsDLZlU1iRp2tgrlEbqu3k4RczBnMvq1im/vfFPtDvsQ94tjlk99klnl2D+fNrDAobnG50eeLZCzy7uEyz1mN1Y4nFLQdGXOy0RnKpWPpygfGLp2WHyLC1TAop3xYMUqhfj5A/QN+VjLfjz5vzoFuF8BpgG3qjdpOSJeVJZbKeRQvKqn4yAjR+rbfPGurWiMjWXfe23k39bq/JejktGGw9jNbjrbJqnCusAXO+ZcNZb6PWHoE76g97XMaPgBZbbpVDf229E66snzy9Es05jQ1rDMjDa/vE1t3WUc+xoVNZT6fKZD3P+k1rZi9zjdaQDoHVJQPOPtfeR88Xgykx18WZc7LGz9WU8ixbyo4THU8y59p62zGlcZ0NXbsZssMM2pFxoj1fLFjt7VlmzbVIlh2isS7Hj0AbrWl2bbSf7N6c1aVsuJLmlwDwPkGHyc5Lu3bakCA9R/e360SaudauqzYU+0aJLYP0iE18Uv4EOgl0Bh5Ua/dJ6dM7D5M9x6RZuQgnF+F4xzltxChf8fe5EQxF2z7XKgLbLyVwpplQb26RtLe4lECtBfkiDPrQ3ISNFVjedADSKVydVhnNj6l9wuqlz8UQsjLud+l0ch5G5v/aQ+1zsvuG3RvsPLH335GvXcnqpztypWheXI11faNlhxn0NS6PnoYTA/j043AqgaVl+JgHhKwoTwSMsg924ZhAu3FKzTyugfcAB6pQ7DnAaTmBrcTlVWngjJgF3IbRwG1Oj+MYFVs4r37s7y8PVgFH17W5MVZwRuY8jkZbJnj4NXHy/ncpbjUcQDWDYzZcInisFwjJkyuE154r78+GL/MyIURm0f+V10XepBoh8bKM/aF/9hSORfUwLudLw5/bA+4iKHO7cEqKFkoBCREhKfYA+D/WYCKB/34y4fZ/f4Knk4iFZsrBs+fYWk+op/BoMvoqbHllICh+YiPAKJNBuUFkAItxs0XwNKp8kglG30K2Pgx5TlYZDWVq4RSXnG+Pf9eA3/hrKA7gD/7wcRajhPxGysSvPUR3o08pdmNkC/gi4W0rJf9cKcdima0xygbqEcLHxM4SgCXQQUp731xzZOAUsgHwGCE07KumvZ719fnOGkRN2EzcOKvjgKuU8JaoJmGsy9hQyJo+1tMW4caOFP4+bvyqT6TcCYCxbBubc8gugi2CcSJFTqCljAaFQum4QATdR30pJTDv71vzdd3EMak0R2Kcx11goTyGA3OvHIGRpBw1aqsqIS9Tyz9Dc0+grg3TElipdlAby7gSILvsny324yoBTLTAkUJcNSc6hISyljqvNpnArRPjjMeUEC6rEFQIYNc6o4ZD4utb9m0wjxsDcwQAXGGsEIwyGQI1XH/g6yljdJ0ALDYYDZlTm1pQVe2m54lRZ43KKiEPhuaHXQstCCvvsQ2jvFmywwzakXGygdu/tXaIeaq8Pdvx0FpmUI0AQFcJr4wXU7WDmzcNRsObrbPGGgUWoBE4pH1c+6L2cgtiaw+MCGFPWoNKhDdbWUeRwuPkIIOgOyhsauDLv8GNA4QEUFnRHtHxfy3IVgTyDch9FdJnfPhVH1Z7AdiTc0bAxY0QPWO7a0gHl09wHac3FjrQv+hyJA0jGKaQDN3HOpf0PO3ZNQJTDALgmWWibUeUEkApA6wTAIJupbJY418OOwj7gMaSxvJ2wbMdeflI1jG3I1dKFmy/FbLDDPoal/N9p+ycbTtj5fwgvGnASi7zfw3KAiGkQAargJAO0EghSsMbyS4RPG1WUZByJdbKOm5DmDXPlJIiNoIAkzqB9mqRU8s0kIfLGmsySrXxi+Ujo1lggFV0rPdbSgOEzapv7mfjyXVMnsAazoib8HUUY2QTZ9TJgy8FTkkEbW4XeWMGwIpXADZ6kC51OYFThrqd/mUFc9m0STYRoDXgLBNC/5eCZz06oi0WGW2vKcKryWX8KixH9dJ9bfspZGeAAw8HLdc+m2sdFn0b5JZal5VqgSBSytPMvTQWbAw/5pwse0btahN32mScJaDix3NEUHr0vD7htdwN36hS0nPmPn3CfLHlVD+orbLhAXbDzHqN1SdZ9p7qKVr6OO+fNRDUP7afswwhy7YZmPMs28QyVLL5LgT2CLDVMY0zWz5LFbdghI6p/GozlVHz3Hq1dZ3Ka+sEo0Cm6jnMHLfzZJzXXuWxYVGWNZNlGKm9NZfzjPa1ZdpZ0XnjGHy2fFZURsuasmOezDW5yLdXOvqbHYPZdhiYY5ZhMM7AyDITsmFsN1N2mEE7Mk5kUA/MX61f18oKsuxA7btamyxwLH1CBrU1zu26blk9un92HUjNMRvabfeSOPNXAJD+VgjOFOkhClstcmUOHQFH0jcSAmh0PfMla/xk1x3LerL714BR5rb6KgaSBIadwERvEhLfv9gybi3L6qbbaRftf5t4MCmFdh96/dExci1iHTnZvcmOoauV3/4/O6Ysm0zltmMy+z17P6tbWp19Bwzaka93uVX6Eewwg77m5T8SQpW00YvlYRd0LdAydAT8xDiDfQkHJkl5+nbgQ23HNhJtV5t0jvA6dYDbcblPLPsnJbCHdN4e4FXAx31ZNvzz78QBQs8QvNgDnBIjercYCFJeNs2xM/7+NkznDv+8OV+v2/09d/tnNYBf9verMPrmJv3VeWJ3JDh2xBEcEPQmX8/7cG95eDXwR8CbcSyZPTigrgC8xZczxuUpyBHYTnVCnpLzvq3bOG/jEg5E0SJSI7A/pJgKuBoSctJIIVUuD40P0djlRdyN6/8S4RXnD/jnNn3/RTjA63ZfLwiv6BXbTH0lZUChhSsE5soWQXneRQDJxIDR+LRvV+r6eyiRJuavABrlt1nz3+8keKsKOAZX3/fPzACm0gB0CTxUv8trehb4ZBO+lDhml+rb8P3zJoLiL+bJgBBONknI3ySvWY2gXNtQqhl/T8mAALSq7yYJ7CkLjKk9rYe74J+1hAP2UkbnqxS5ISE/jPIQKaRzggB8aZ1QPqQ6geGjcTnlf58gsNcEhkBIPr5pnm8BF+s9lMdaTKmO+V31k3dfymYLl2dKuTzEoNlilCFVMvfReJVo7VgnjMsBIaG+XQfA5SBp4eax8jEJcBVzaZqQoFbMMD1fLJ5N3z5bBEZbgZDbyYKEMo5if+9V/9sUbgypXSeAg2VoJ7DUDTkdxJSDkBNpnWBYqe9lMEa+XcWIUztozbSg2zQuQfsgCW1+s2Q7Xi+dvyNf+yKGIIR1YjuGvHUoKFxY+40cZ3oJAYSXT2wyysS1crWxWiA4YQS0a18UwGMBWgsCxL5s9lPBzeUqgRElsMUyj/LmfIXPruP2wFMER+B254zaTAmXLWNZOgcEdq0NRVconF1D5LTRR+e8UJBBYL8V61iLzTHpXdq/Fbplf3++caWxIwbrOFDlua7VuRoXlg0tHVj7hAxAlds6RFVmzG8aB0NcvzQI6RvSzH3FRtLzrANBY9CGsEfm2p3192tHBCZuVyyQ/rUuFhC/HubeiyHb0ZFeLvNzBwwycobxzIKsWPTfblr6tHAKgPLDDCO4MHS01jX/m4w7a6hAMDCkZGRDeGyoyyRhEZAnXQoDjLJmLIvH1st6Jez5AkMKwHwEu1LYH7vzjgD9CA5EMBO5Og37V7I2rPKWM//X80WBLeGMzWn/qeCMQik6KQGIqeBAKRn8AmaUg6VIMDgF6On5aleJqLpiUliFUYwbu/mLwTLhr4/8P6UYpiJXtpnYva1jNopYjSKORTlK6ZCT3YTeMBi39RjSJJTDKlDWYyYAJCZ4S1W/rHfJjlkpDaJIC1AamN9Uf+W7sl7PpmnrRVz/lGKYjKAbwdEYSn1XljoBdBCjyXpgB8DiMISyCaRQjh+F+2U9cVIexQBT31hgRAqSrtWCZhdgq2BqjGDaKutltWPEeg2tl89u2rreJibWfey9Na+sImcZPFlWiNpR19txoXqpzzV+NO8s4CFgza5R9pnZ+6pN1da2DdQO+ms9m/YclVssP8tcEgBj1yF51wX22Vw5ak8bnqnjeraAI5VDxk2WzXU1RcuOG5vgU/1QKbiQg7g72v/2vrZfJLG5R0RIoq17aN0ZmHskQCGCOBoNKbhZcj1e+h352pc08307/W5ZDtIr7L5nWbFaIwSqNrb5LBhdo6LM/y0rqWC+Q1iTxQayOYwmcftglbBOaX3Vul6MoOI/mrelxL1xcwX32a7YfUsAvNYQ6U5Kth1j2DL+egs8WABB66Y+WcaUFdv+dr/I7h1qX3s8NsdzmWu11logBMK6+VzrkNW/VcbrWYtURumj0lksA862nfYim/dP+ySMjh+x2KX/ZgE3u39m+8fWUc9RH9v+sfvQjrz8Zbt9eT3g0ctZrI15q8b9dnSkl8vc3AGDjNhOk7FiQQyJ8uBIkRft9iJuI1a8sOiqtSn35gMxgqx3LSWEFnUIIVPnCTRj5ZqAMABb/hx5MFJcZ57HgRUy8uUxqfvrpv2x2wlG+BwuP9EMbpKt+uMzuA3tdROQ68NcHbox1IZQmoSZGpSLcK4C0achSYNBp41dCl+bkC9DZV1m9C1APRxzZMnXYwP3ZqkGcMgffyeOWZP6urwC+LIv9ySwK4KJPHT77o1s5whesmlfljtw3vmEkDNJ7IV1//+jBGWxhmPExDjg6yAOFKlEUCjD/B6olqFSholdUJjJUaiU6VdnmartprN1kf/9L1bZONm53OaH52C+Aatt119rBLaBNYrFdBDIIq+VFGqFGFrgIIdjC635e03jlNpDwGH//0XfDl8Cjvl7zRA8tZf88ftyMBjC2/NwbK+rY1yEw7vhfz4Gb16GN0dQLsGHO46RdhCnDJ3zz6zi8ggtErxkTxNYd5M4Jfk8o0yWph8zVa70pkHwKFovqCjxYnxNE3JNaD7VGZ1zmo/yDPYI41EKbNmcI4Cwwmh4JARWkOasFDnNdSmZUsTF1lsieHfFjFLS+L5pA9XZAiHrpg0jgse+juvPDQILUW0gho0NDxVLKE946+AGbvyrbVSWiBC6IWB2hlEgW/VRDh4IIbB6O5faTECk+m3Sl2XVnBPhxvUSYc3TnITRvEq2X2Xs5Rldx9UWAp5UvyJuTurZCZCfrlDtJeSa3REDVoaPyr3LX2eVfIXCisVmFf06gRFRNW03xJ2cDkffinYzZLtK1stF2dmRWyMy+m3olQ3tlMPCfte6Po4NdC1ijXsZ2tpfq/4jNoj2AuuEUFiYwKAa4aUgkwTWkYzzHFAsQaUK1ar7m4ug23O5J6cabl4v451n26yHyjDhj6lttG/vx+mNW7g9b4HA/la7Sp9RTjitdZYhJP1UAAlcue+qPAqFs8BaMfO9nDm3YO6tv32cPvYMTkew4bXKAzlOlKtRe0nEKOvZStbBpjGpT5XQt2KNWYaQzR9Uxemgs4RckGontZ8AIeloyr95gZCDzuaNy84DtbnYSdIhxLjW/rez9u4I3BpAyIKU17tOX88zbXjxrQSDdphBXyeiyTUuPrdvfpN3HEbj2q2xVihAJ4J2eiWFVp4GdYQ2WusB0G/WA9AnvB5am3eO8NpqGM1No8255n+fBSYiGKQuXOtA5JSISWAhdffd48tyXwF6CUyXIc65JNj1GtSmHRjS8lQZbWA254bqkKUpW4AohwMC6jhFqUl4O9UiQYnI4cAFa6DfhgOQtPnWgHwcwKwFAlgnQ36aAFJJERBlXWE++4Akcm00CxyLXIz9kn/m3ggmclAqwr46lKtQnoDKAcjrx3oVpmZhdZ3iZ2K6BOV0ogSlTugbC+SIDSZlxHqFxOywY8YCRULNq4QQIyliM/7vXn/uMVyC50P+mnmC57GECxE7Ernr7ozhNVUo11yfz+6Bv3rGtcP+ItTLEV/ppiykPv9T5Lyh0xFMpoEpJ6VmnaCcq67NzPhQ/a0yqPqn5v/Wo6lzZagr5E73hDCWshtobI4LFLJzyZZLx7IeU7E8CpnjWgv0V98F7HRw4z8iKPBi0EhxlRJr1xdw80TXaq5Z8EP31AZqvZnZ+mteiNGj9lI5BPAKUBJjUcw5haUKSLceUxhlKtl2tPePCEaG2k/flVjWerAFsmiOax2SJOZ33T/7XJ3TN/eRpEBUzJFPoxEjBq5c32Ro2T6WwZFNsKt6aoxrHbg8vqMwF2+mWPbYtZ6/IztyNRnHDrFhDVoTFP4k1ugLSZwufUjP0pyUHlTMfNe6rnVLx0vmY1lCVQKAnccx+SoxVIswUYXKhNOVum2I1p1zpcb2FW5bD7GqLOtXgNYkDoiuEHQnrcUCFWA0MbZl6VgmpNUntTaqj/D1rhLC07RW2+8CgiYIzCWrY9nnaR9ZxOl/1oGS1ZetqJ8EBmVDoG1d9H/7seNR+1eNEOInfUIhXNp3J3EpAfbgdEmtzwmjbwzTGGv4Yw1G30xsUxPAKOAGoyxf/TYwx3ZkRyS3gh1m9aCbFa6YncO3SrajI71c5uoOGJQRIfU2tER/7cZSwhnV4Bb4u3GL9BSBsrsMfAWIW1CLIE3HDwyh/GV//VMEJoCMP/wxGdTaKGLcG7cu+e9SVO7FsVuKuM12LwHoALizBLOTDtipApN5B/BEPdi3Du22y1lBAs0OLPZgbcV5uxYS2NWAdBGKOehOhbpIodNGrbc/tQib9iTBu7Hhz2ngNtbE12XDf57AsX8a/n63l6GYwmbX3e/1M9DZhH0l+KuWA7W+MnB1ncMxep7F5SLaAtbzri9KfXglzotfxXnW1nHKwGHgPSWYmYN6HSarMFOF1bOw0YT5GpRSqFWgUIKp3a5DyrMQx0A0hH4XWmswHMLqMq/u9vhjnHKwCSxuwVLvykVNLAAxiKZwCpL6LiHkQBL7RAZu1bdhk8D+mfB9PwF8U9GHuCUwOYBXVOGplhsn5xl9Lbu8VC3PTOglsLEKrS2IS258bG3CnfPwmnfVKZaLvPmvlhmedcrUvoNwbAsOlOFUC5LN0dCgCiH3iqjzdkPbIHg+Vf+yr5vNAyMldN1fJwPCAqoStd064xdzKX81Rg1wMU3EmNG4FqiRI4TiZT2rVulU25YJ82HLH99DUKIFQhUYzdOxSVAkrcKs8FA9p2fOlxe77evUNV5EZCMAAMbXSURBVNfO+7ZVPqyGadt13w4CkTYZVZ5tAtmEwJJSe4sJY8OfZJzkccr0kwQFuuXLoXLmCG8rlNHSxM1NgYiJP1fjNiW8hazsnz9FYP0UTb0GBC/7uqkPOO/tBMFAaAH9tQ6lQXq5bGInKheSgKaWf2aRwD7U2BG4LcUpxc1ZlTk1n37k+k65hm6m7IBBO/Jiig3jFAAuAFdGrhJQ21CZq4ldTy3TGsI6I8BD4LH2TgEUWi+1RkufsnNQxrfYGWJX9gn5xbS2VFKY7sHMBtTbUFl1vzWGcKbl3rZ5HLfXbXduKel0h7AXWsC9Zc7fhWP87mc0CX82Abddc7X22TVQAIkAIduuFYL+IXAHRo3DbG4oCxTp3hZwUb9bIEfnjAMFxXASKKUxlXWi2LFhmU6qo9g7YgbNEnSLLcLer7YRK0vjeIvRN8xpHJf9veZxzjTpxUOC42bCP3fFXJ8VXRcRxuC1ssputdwKgOJrQbbbZtbReDNF8xi2rzNcr2SBoFsFCO2AQV8Hkt0w7OCzzIsiwdDo4vLo9HBG3QxuopzGATtRF0rR1allMua1cSo8JM38LoqtFCF5s/cQDGttvPM4kGgKt2HtJ7xmPAVelYM9Nej3vPe6BJMz0G1BqQlbHRjE0E2gM4DVAbQG7tmngF7LbZIloNgarZiMeHnj8HVS3hKFQ4gGrNwASiK4RQg9WcABWx1f77kClBM47N3sh6pw+xbcVYRHW46JsjV0507iQJHI138BaMeuLwqEZMgF3MYtz98McE8eDk7B1C7PgJqG8w332+Ssyx9SrTvvX2kCBl3IVb0SmaYw6BP1WqSDHlGzwYHB8LJ3tI1r38bgSpCx4v9KUVKIjZQjLfoyeKVEyOs57dtW9ykR6O1351z7bAyhPYB9RZhpubpfJCgdCjkbAN3Uhz6l0Gw5MJAS9DvQ7kB1PuLgq0tEpRpHH1nmzFnXn/MzsHcAcxMOPEw3RynaRUIyRQF9ti3U31J2Nf7FPFP7SKGTSFHS/axCIgVOAEtWqdIxKblSTgVcStHUoqmyyKtcYzSxt55vFVO1b4fw+nX898T8JiXbMlkUKmQ9q1J8rcdV9VWoV5HwCnmbSHmC4KlUWdSmAiTtveRNFhNGQJEFZ1R39a9CoLRuiXkzadpdhojYNTa01HqHW+Y5mguWWaM2miKEqlZMe0KYOxGjSrzWIwFKMwTv7hYwaA2oJMGAnGTUE6y+6OLW3ioOYMXUS2CRygGB8ZUNYdNanj1+M+Rq+9Rznb8jO/JcojkGowwIrakysK9FcZYRomvHsTN1jmUVaz+dwM2tLDvJPtsyOLUXKsedACLtCzl/z8HQ+YHohDCuDdzeeoHwIontzi21V4fRdV9tId1xGqcL7ja/yVkhh1uDkJRb+4X2Oq1R2qNsXicb9qVwNQFtMAqkCaRSmF4pc7+8uZ/Wd7Wz6mwBqCyooD62oYdaq+2+ZK/L6vE6ZkPcxPrKEcKxmoSQaltWy6BVCLNsAXwbiYWvvJbTmXYXs6phrs+KBVK1R74cJNvWO3JjZbt79oshWmfl5LwZ8lJhBm2nvV8uc+AlDQb93M/9HH/4h3/Ik08+SaVS4YEHHuBf/at/xd133335nE6nwz/8h/+Q3/u936Pb7fLggw/yq7/6q+zZs+e6nqmNxi7MdkPd7X+fxhkMVX/sLtxiLVZBB7e5JMBKAsvplQu+WEgwuhnKAFCuH2s053HGzkzkjI5dqSvHK4tw7BA0V5wC8ooZ2JuH/gY02hB3HfCzjt/k+rC54YCTYQKdHORaDqBY6MCJ1L3J5iIwMXDeC8WZbxDyg0RAsRM2t4apjzxTMrSG5rg2zcjXx4byTDIaKz6J84hNAl/swp40sBk6A5gtw4Hb4dVfgnoO3p1AK4JjVfhkwyliYkFMp67vqkA975Ii53vu/7sIxt4zPcitwFoTymUY5uHiElzoQr4NrRT2r0AjgqULrhy7zsD5CIZVKEcpu3NdFqM+BzsJF/vBAzoPlAYQJYG1kfd9LS+Yjk3jYs1FmxddfYjzTk6ZsZEQDHyFxyW4N5kkwLm+U0iXUnfPZtuBlQNccvOYkL+piwP9Dvjnn0hgtueSR5eHUOnC40Morafc+7EG+XyXcxcDmLfRhokOTBdhZuj6bpWgNApcEPNCwIPmTN6PMSlbCoWT2NwDAkLEPNEGlRLAmTrBC6lcCRGj41CKVtG3gyjeAijyOGX7IuFNcgJN5O1T/Wx43xZBsRUzR2wa/H3FjrJ5fFqEN0nZMDX9LiBKANsAN45liICbq1JwpdBaQ2Vo6oK/xzphrtr7T/j6CriVwq0yyXuaXUOrhP7uA7M5uJRAPr2SZSRW1JR/jsa9yqf2k4EmA8kaNKLlS9m31wvQauMUcykVfQIjqUaYh1XcuL3Yh/U05CcSWG0BNsyzagSP84o/R8xI6+m2AF/e/78A1BMoelbeOjdXdsCgl4/cCh1pO2KNeutok1PjWphA9l7aM+SwgFF2kA2nUqhXxZRBYIYcKdqTOwRQ2eoockoovEpgu70GRt98aZ0Jbdyee5KQM/KFSHZuCpBu49ap8768B4DZAszOQHUCkhiaXVhbd6zkiziWovRaOcWyIU52jbfg0DhwJavj9QnMcBjduwTCDHHro4AyrZ8yMMcxfGB039F9osxvsbmX9oLsGBRAv+yPaU9QqgKVyYKOygO0QQgpUxmlE4vxuUFgLq3idAE5KRS+b8P1xomdJy91sW0PwaH1QveIHWDpueVWtM3V9AQ797JjVmvG9bJl7LqTZf7fTNkBg26yfPzjH+fHf/zHeeMb38hgMOCf/JN/wjd/8zfzxBNPUKvVAPipn/opPvShD/EHf/AHTE1N8f73v5/3ve99fPrTn76uZ0ppsINVno0BzpCPCfHZAhCOEDYCbYbKv7KRBCPCDgwZrzBKO9OzujgwYMsfb/p7TuDeRjWTwsHUXfeGArz7MJzqOwP/bxyAYhmWzkFjHdYSWO65e24BrT40N2Ethu7Ala+76Tawc8BDOJDoq8B0Eoww5XWpEgyyYte1SY0rwSDVMasA2rxLk4wyFuRJUdtM4ACNGeDxnlN8pNB1BjBdgN0H4e6HoZ+DXZ7Pu6cGxxuuz0T/rSdQjzzVOQ/FvAODKriwMilDp/swt+4Am1LkwLNFXJLDoW+nu3DHvuzrfZtvtz5QJ+Ve+jwB3A/s3hfqvQsoDV17yjOU8/XTOOkQklvruj6unCWCMX/QjA2BKTmcciMlZsHfZ3HgzrmIC8W71HWAzxYOMMv7czXeOzhGVRsHfpUH7pmT/ZBwe/8GXPhCm1LcZmEj0Ku3OtDuwbAP9WHwkkmkEFX9/ad9+UumnrsJeY+ameu1cFkwSGCllKuEANZMEBTTAQF86pr7CMhQeFbZ/K422UVI9iygokgIPdIcFgMGAtglsEOKTdk8e92UQSBPg/AaedVJyq8MIwFcqq9AMm3CDRygYEMLeua3hNFXxA8JIWFSyPWb+lwgowWoRJ+XUm6vUwiZ+qmeg9Mp5NLg8dS86/nzJn2brpryWmVb64tANR2Tcq+y5TPXqx+GhLVK7VUjGJvWyIuBS4OQf6pMYOvtY3RN13UyEioEg3OSwFjSGLDrntq8gBuvRT9vtCbcLNku5fvlQoP+WpRboSM9n2TZF7H56DerC2RDvZ7rvgoPkl5h2SRiNebNR+dbYEgfgTqWgam5CmFOWi+0BY9sjpq8eYbWOa1HbRzgsX6N9XwuyV6vNbHr71/G7ZuTwP487JuFmd2OwdxuuHWs0oBhOpoHR+2pdV3hU5blqj3W9mUWDLLAhfp5YP4fm3upvTdwa5yYpJE5V+dlJQsyZhkDdg/SvWzYm8Kv9BF4tenv3yG8CCYx98Hfa4sAEGqvgbCPlf29FLKf899bBPaxHSvPxXKwwNVLXdRntp1faAjTDsvo5SFZ0HfcmH2hjB577a0Eg3bCxG6yfPjDHx75/wc/+EF2797NQw89xDd8wzewsbHBf/gP/4Hf/d3f5d3vfjcAv/3bv829997L5z73Od7ylreMvW+326XbDek8NzfdFmAXdXnkZSwVcZvWOm4hXyQoFEOcId/DbQB5nMG9hPdcl6DbvXJRHLe4bRHiw62Bps1I3v0052jJ4F+9moOkD2kPOh1YPAfdPJzfgI2OA4lOA2f8vRdwDJtW4japNYLisuHvaXPXwChYpQ0WwgZuJYsKSykQ+0VGrVgaJYJXvJi51xCn4Mir/gQBvOm1Xbx++iwcTx3j5iDu5utNZ1Td6+u3DzifwsnUHV8cwETigJEFHAgminEbWB1CP4U4CuDIswTvV9/fVwDKCiHPiozqmv871wjKYj72f5MAKtQJSt0kIRfJsi+LpazbRVfhJPq/FNkugUo/wNXxU/5aATYz/t7yWPYIxq/6VsyLAcGgl6dtDuil8PtdiCK4lLr5sQw0NuFiD3pbjllWZ1SxziZdnCQY6Sqz8hxYkER5XASQCigrEPItZcMOrAe5RsgBIcaL6itQRHPPMjhkaAhskWdaXsEGYb6KBSXmj+pQwfWrFBtdL5q5fisQWHWac1LSLeCDbwt5MKuEMFGbR0JgS1ZsaFfb/xWzqEsIz6ua5wnQghB61WJ0vFljr+TbY53QN7kCTA3dNQpbkEElQG+VAJRovDQJrJ/UfLoE40Vtsklghmm8yuiUgq8cSpMEwE6hrNodLDvKPlP94IDfwIjTuiAQz+Z/soqLDICWOS6WkELHZHC2ubmyAwa9fORG6EhX04+uVbI6jtZZCGufZZFsx8Mq410guO6RNdjF6pEukfPfdY3NwyPDX2uRAB4LWqj8mt9aI/vm/lq/BFCojirLjTJmpUvh63DcP/N0Hw6swHwXyjFEXQcIrRsgyLad9koBdBY00blyQlQJDKg0cx/L9rL3tKAR5rvAMrHBczz3GqS+1/oqZmkWhILR8WWBJt1Dn5jRcGLpQ+PGqNUZLONNz1MbqVwbhLVee8QGo7qe1evG1ffFYNfcaLEgnLWlLCi3I88vLxfgy85R27/P1dd2jFyP3mCdDHDrAIwdMOgWy8bGBgCzs7MAPPTQQ/T7fd7znvdcPueee+7h8OHDfPazn70qGPRzP/dz/LN/9s+uOC7jCAJDYUjIJZMQkr0tEdgx8kT3cEyLGg402PD3ypWh0722QSEmhELDrJcfRsGgfgrp0F+Th+HAgUHtDbiw7sp4HGdYPQw8TgB9CjhWUZS6Y2uETTjFGfrrBHADwoalDVOboJgL2QXAllub7zSBhVImhD4JTJlm9E1D+Ov2++s6ODCoiEuM/HTLgTyNJ9318304koMoB6sN9wr416buFfX34sJTnsIZcBc8cPQ0Dgg6TVA0TwJfHgZjcg7X3yf8/ydxoEcL184H/d8J//sy4bXca8ChLfNa1xyUIyh6RlLX31t0773+vn0ciCOQQIqLNeJl1MpLJkVWzJaO74cL/pMjJDo+RAD/ZBhPEZgmMpxFc5YnTON4N9BJ4Te77vd7/PFFoLvhlcU+vN7f13rxxLoRGDRFAFgEBimflIxuefcqpi5qkxIhaXaWUm2V1GnfpmKYWO+i6invXSdzD3l/LWAjYEbhaKl/xjKu/hCYXwo5gjCvNSZi3w8Ch6SYytgQSCnFVQqD2DObhFwYWTBIRkvWiyKGmUBEhUhJMRYT0YajaY2Qwqv6SbHNKuEVXN+ewrCwCi7kMDd0YQ0r/rjat4mbSxaMq/g6bjCqoGusyMuqvFkb/vdNU161nTbygn+WgFgxBwQ6yRiKCf1ggS4dF1tUSr1Ybz3CW32U10LtkzPXTBAAKAGOmsMtRvvyZsh2FfeXg+L69SIvho50Nf3oekRjKcocg+2HTAkM0h43ZBR4sABTTNDL9Byt2VoLFcakj/LDaF+CUdaG1lyxtqWLWQPHMmkgrE8vNDzsucSyRbdwe8+zwOwADi7BniW3Bk8QQArpFwJ5LEA3MH8FlCnRtPbBCoFJmWWCWQAo+z0LAup8sXAsYPNcQOE4MMjWxYJCWcAwu3dI53sxRUwg7aECHpv++yRuj9lFcJJcDQzKAp0vVckCZJoX2XbfrrwQJsnLTa6FBfVSAYuyzjF73P61Ypl7L+S5upcN0byZsp35+FLoq2uRlw0YlCQJP/mTP8nb3vY2XvnKVwKwsLBAsVhkenp65Nw9e/awsLBw1Xv99E//NB/4wAcu/39zc5NDhw5dRrPlQS4RNn67eTTMsWX//QyB8qqNMxR+PJ1NG4AGizz0yjOhGG6JmBMDXDLnzcQZv+vAQ13oXYLzXXgqdcbXJo610sExleSdUnhJE7cpydCxC5H1GmH+n8cpAcqpYhUisUjswqBJrzrK22LpxmpvhXR0x9xjogzrPVhNRsPo+r48X03CK2kfTl1I00WgnYY3Uq3hlI4K4S1cK/64VSDkRVTYi5QJgQB9Qs4RKTBb/neBFy1fHwELpzBjKHGAhHKUzJl6Q2BZiMmiSSplWIat2lMJenWtmBYy3u13bdJ9wlu1lOhRfVDydRPwY+nh2tBteNUB3DhUmaRgyROmsZIjgF+W8XMYn5g6DWNJ+VPE/JHhLGUWggKsBI9iC0EwvmU42BAxMdLUpvIYY55rPVsQQJ9lwviVES+lWv2puS7AQO2vfrIhBJIJArNGzJW8aWcp4mpfS3sXCJMSxrXawM4jWxZbzgohZCzr1YMwD5QrSO2hdrQsMnnIBRYJ1NAzc7jQQRI3/tQvqovaTcCUHQNiKWZBZstIEsPKGhoTmfprngvEFggl40dAXRYs1JjS2FVfiVWlZ6hvIvMMldV6tKwBoLVF42ADN382Gb933EjZAYNenvJi6UhX04+uV6yRCGENuh4lXroJhDU0O16tMyFnztGc03wVmCC2kfQHy6S0YU2xuT/muL5nAa8o87lR8yQL5rQIwLTWxhbhzZIpgc1pQ6VUd/v/jjlXepr2ba3Jdn2y+qMF66zBaIEZnW/z8mQ/VxOxd1QPlV/PyAJT4+59oxg3Avct6KYxJt17kxCSrCiErLwQEOVWSJT5q+8vZA68XOr+9SZXm6PPN2ZfjDXxVs+JHTDoFsqP//iP89hjj/GpT33qBd+rVCpRKpWuOK7G0AYxQYj7FVNHhqxkzf9dZLTTR74PrjSw9BwpK1I6GrjwnVOMvm0Igse/C6z1w6beAr7ahL86CYvp6GtP5TVvM2pkNXzZpwgMCAEQMnC1WcqowV87458p9pRyYCiO2ubEUL1kwFoqsAwxgT8KBSma50lmp+DiGpzvOaVm07RFDng0DSyJpxL3mteTOM9LGWfE13FG1gEceCdl5yIOFBLTQblsLhEMUfWVksA2CDkDwDEZVN8p3xYyZEUVlmLSH7qy1fw1h3D9rXxMClkZ4nIdicGh8SfqedFcI4VM/aq+k1FqWSw5f/2C/72KG2sySKd8e0hRiQnMC4Fb8nDFwKsYTfQrWrzo3xvmerHBWoSQwddEMIwhHbpylv25Yso0cIDZKo6FIdBHXkkFMdi5Iq/lBgEgU0hegVHafpUQVqTxaPPoQEg4etrfo4pjoDQJa4LyOcXmWs3viJAXSM9QCFGC6+cLBG9iSmCIQOjnvLlOYYe7cGM48W0kBX2C0bVK61tizklx/SD6uvUyS1lv4PpljbCJKzeVGEya/wKLFXI2JICOYi31ulBIXJm7hDAwCzZqHK77uncIY0chdGobjVOFaWTZVxo7FhyEANyKEdQihGxq3bQAsIDZc4S1Ujmj6v6eMnoLMURJeOMi5rmqrxh/ausSjm3XwK09BwgM1B3ZkeeTF0tHupp+dL1igWsZ6JYBsh2xe2AWUB93zpDRtyJqjbCAhQxXsfq0xqjsmtOqQxYMuppxk5pzLEPpxRa1gUCVDdzassqonlcnvF1SOowclwJVFJqt79Ivx5VdILoN/9YaLqaOBWK2Ux9raF4NJJEjDIJOIYeL9G3psuPGyY0U6wSJGHXmQHD4qM00XsfJzSz7CzHQn4v1oXpu19EgebkY1C9UrpU181JkB9lj48CaLDh+veu/7I4bBeR+vcrLAgx6//vfz5/92Z/xiU98goMHD14+vnfvXnq9Huvr6yOer8XFRfbu3bvt51ivs0AAeZCzSLdkHACkhe+ywTC8eoiYvd56jeuMKiIa+AJmNNlWCYblVho2dik9lpIrpShvnqVNPiK82UlGVGz+WmDHov9Fc43uMTD30KYsZUqi9iiY8stIsmUCyBcgF4f62Pj1HiGPyATOcNrCgRAyRAViiB0lL5dAJykvYmKlhDdsaPER6CXgyjIrtKjZc237SnGRQijj1vaLxl3enFdn9FWnahvribM5XOw9ZcSqfW2bq61hlJ0l5U19ZnMO6b6WXSHQZsU/W/lmxDbS+RZYVHvp/gMgTl24mZ1XMYHWP26DzG4oGqsSAWFivqn8tu72WRpLAkmqBA+y2gdGlbg2o2NVnlIBT3aT1NyS4gyhbXuMMno0R60CbhknArRSc4985nrr9bTHydxH18s4yoLdFqy2c9auAeqrZqa9ZIh1GH0LTeI7wxpjmtcpDphRXgoxcjSmBK7Y9UjlsOxF3UvMODvedY7Gt+arAKWh+ds099JeoDVZ7WHzmOj/PbOwW+ZkigOVIkbXsyz7R8+6kUbk1cQyQq/1/B25tXKzdKTrEWswjGMOXO/9nu8eWvuy4UN2XdRaHo05F0aBH/tca9xmjeBsPa8GaGxHsvudvZ90K8tc0rrWIrwdUuutUh7YHEliBHXNd4EoVyu7nqO2yLKkrmb4jdN5s3q01SGfS9QWAgZVHlvGW2Uwqk3GASB2LDLm91slL6QcaebzQu/39SiaSzcbvLweGdff2d+eS17oHmAZhjdbtqMjvVz0o5vNQN+WpGnK+9//fv7oj/6Ij370oxw9enTk9/vvv59CocBHPvKRy8eeeuopzpw5w1vf+tZtP09AAYS3bil0QB4IGdkyJiSFzHd5iwtAo3f1ya2BYgd0ERc6o2coQbCMMsvKOI5jK3QJ7BPlrFCeG8mEv0Y5SnIExlMOx06QUiHDViAI5hq76U8yClpNExL9ChxSHSYYNRYjnGG0TsiLMU0wzsRSqFQhn4daFDwuYhc1cSAPvs32+/Ke8Pc449toBfhrnJdMANqUqZsMPt171X9XGFgbpzytExLEqs9lcGqDVyJs1TMmJLNs47z+GwQgQKFTyi0iJsJhXB/m/fVT5nkQGB5iyqzjmAQ6p0Z4y5WAEIFnWY9ngxC3L9CwTAD7BByI4dPHjZdJXH6CFULi3z5u3E2a6+V17ZhnKzdUgktkfhksILyW2zI+ZDyn5h4ai9ZoFgOthGNXjIRXcuXmoTEvZluB8JbAki+nWBybBEBA+Zw0F1v++6KvO4RxMUsATVRmtccmYQ5LyoR+gpDcWAwlsW6k8GsMWeVTnkmxAi3rRqGA+l3r3Boh3w4EgFQAkMAa9ZPqZEPVtK70CMmjNV9ioJ8E8EVzvefL0/BtNUFYj8TUqfr7TxPWYIUyiqXUIIBIEFh1Am903aQps9Z2tZeYX11CH4vmrzGiuaFQUDGrBCBt+QZU3doExt4ewtpowVxrcKr/KqZ8N0uS6/jsyK2Rm60jXVcZudI4lNHzQsSCwePEMn2Ur6VBCF0W60X7vHUUWeeOzUdjHX25zEfrkXIgao2w4Pn11lMsHOkHFtyuEN5eaEGsHo4RfRqnAx3H6UWncbnaLuLYwRdxe9Yl3P69QWA+P1c+HcvMsu1iP9KF1V5ZJpXazsqQK/tinNg+ErBiw7EUPnar1qcsSJcV6dbW0ftyFqtjjPt8vcq1sn1gdI5cTV4q7flcIPe1AEHbaZfsdQKMb7ajTPK1qB+9pJlBP/7jP87v/u7v8id/8idMTk5ejnGfmpqiUqkwNTXF3/t7f48PfOADzM7OUq/X+Ymf+Ane+ta3XjV59HNJFmV8rk1Innnrzc56eDVpN7kSDBpHkdN9FXphjSt7PykAAkTkZbYsCSkk1hNexG3yMuxlkNnNOGfuEZvr9XwNbhsKY5+rsijMS99rhOSuEwSwKgUaEQwiqKbu08x5dlPOsYJmc5D2oZCOevQVKqN2V/vY0BwYjV1XvZXvpE9QWFQHW0f9FXMo9nWpMAqciC4tJoRVdnSNGB3K87LGlYqmDG+1txY71Sc15+rZ8vRB6GuBgDbHi+pvNxsLaqqeCkPS+Rbcs20ogMkCinaj0vgTGJSaZymOPiXQ1a1ICbWeVwE+2bqq7DDa5gmjyuw4r4tADQGtqnsWbILRuag5JlBToV9qAwEQWZab+s+ywTR+BNro/ta7bMtuz5EnV2NBCmgWULX3zJv7QkgyrrJbyc4rzQPLLLLrktYtlVllwxwvelQvn44myLZe3XwMg9SxxpQgXc8QsJOtq51Dtr01zrQG2lACXWeBsr55ll3/IDAKLYPP1s0COvnM/7PzQt/FGIMASAlUtve4mfJSUDZ35PnlZutI1yPWYB83H7YrWgOuxZCwBiqMzvusMXKtZbJAjN2ftF7oGfb5dk/K3isL9uijNSaPWw/zMcSRyznYHgagYwIHmGv9smG+2ossUCJnhv2/ZQFpj3k+r3uO4DzRR3uL9mndw/6191WbyOlm+2kcq8iKvdfNWK+izN9xx6zOIr1S5cyGq2msXIvhrHu/lGUc+HMt4+ha5FbsgduV7Fpij19L2bN658uBHTRuDbbrqh0L1wP+ZEXjSevGrRwTL/XxuF15SYNBv/ZrvwbAN37jN44c/+3f/m1+8Ad/EIBf+IVfII5jvvd7v5dut8uDDz7Ir/7qr17X865l8mlznWP0lZYHcV4XhSnJ094HnmQ04a6MQV0LYZLUcfljDhEYJjZZKzhg4WAOTiQwnYZjOcLr4GcJOXR6vryTOO/PMkExG/jzEl/Gsi+/wpPyjOZkaRPeVCUgSUZf3Z9T889I/X2avn36wDtxDJe7IpiKXRLsUgESJRvqw0TFKT3lGkzPwzOLsLgKtb67VkpYCedlXyKEqQiokWEqhoWMWF2nnEBNX569wDMENor6SLlK2r4NLwFH/bEz/p6buFw2VRyb5ysEUEHMnGO4N7rtAaoxvBJYTpwhKFDIshLwZVsmKEsKlxHooFwoh3E5iGx42T6cx2/Dl2Pa96vqI+O1SniDkgzTZcLbm1Icuwb/fYug6Chn0wqjAJRARrFyzpvflGtm4O8/wHksrTc29vdUfaU0lU2ZIOQXEginHDBioykBr4Ce2JejYcqZJ7xBpu77omB+U9sIKJkkgHr7fbvrPPWdwr7qBJBLSrZC/nbjGDNSHA8QXk8v4KVEAARkAG2ae+UJc3nC/60S3giXJyS1VjsKQBVIM8TNnyFuzZBBILaKTdqsXGM13w4nzXN6vt3rhOTpOULum1Vzz91TsLHl8nrZPFMQAMZSGdaGkO+6ubVBAG5LhLeXVfwzpHApF4+85H3TrnV/zVlGAVzlz8C33RnCeFPeKfVJF1ceKSKqq1hHNnxsgsBss2+mq+P6Hl+ecwQG0HnThouEkDl7/Y2W7RpX16MU/cqv/Ao///M/z8LCAq95zWv4d//u3/GmN71p7Lkf/OAH+aEf+qGRY6VSiU7nZrbKS1Nuto50PaJ5mPXiXq8yLdbokFE25TixYe45ArtSDiOVazveW2vEZ8Ggy+ANYS1QeKjWVVtv7ZvTON3gkP/sxa3HVdxr4UsVKJYhl4NOCzabsJa6dWQLt29dIoTG27BjzPPFmokYfZOagArLvNU1V2uDCiE0X+2qNTXbNvq/mFZ2jWnj9odVwhjR3qXQ5XFiHXw3WqxjxYKJMApOyrkX49pGzPkWbh/ZIrSLWF5wdcPWOm/GsaRebKDohYAuFvixAPAL7aOXCxAkx40FVLUuXAsopggK2YgvBzDIroWauwKJYfQFQ5b1ZFmW26mn2sWmAbgVsh0d6XrH7s3WkV7SYFCaPn8zlstlfuVXfoVf+ZVfuQklCoNaaL++WyTcemS0scqoEkNjypyXBwoRTBWdQnCuO+ppFgtB3SpQQ6Eh2oTy5lxRpGE0eaPd4K3R2SOAJD1CosEYn7/I7zqFCHbnYOg99+XIXbPUdQqNQjS0QR6MoV6EY0XoF2Jum9jFdL7EbYUStQi2Bn3K5SJp3Ccd5kn6MFlLycc5SlWoz8F6YZ1ufp3aUpddg1B+GaGqg9pTBpQAICt9ghdKbWeVNy1UAhhk6Evx2Y9T1CbyjrWUFmErgrkYqokDvToxdPMFpxQN+gxS2LcFZwcOpBFwp/C/ccwrLa5ShMQEsmNCBq+YExqLJWBXDlaHYTzYkKeKv6cYShOm/v2iSxDeSz1NewhzKdSKkEYwG3vGVg729aDZhqk0tLPaXSFdGlt6njbMmAAm5ggGt+aRFnwYDQVTO1kWi/XQZll0PfNblDlXCpna27LdxLrT/5VOtcIo603XiaWkvpACpGeJQaT7iS2D+b9lvKm/BfxlPSF2jdF4tQZJjtHNV+PNMq4sg07jySqpCs1SjptJwvyqMhpClvPH6wRlOMIpwTISdG45DknurUJtAeo4hmEyum7ZNte8tewgfR+Y+17Ne6z+svmF1AaY9lSS7koeiinEw9E5p9/LhCSgdjzYMazxpfFjx4cYS5bFJMDvZis8tl+u9fztyO///u/zgQ98gF//9V/nzW9+M7/4i7/Igw8+yFNPPcXu3bvHXlOv13nqqacu/z+KXqiP8WtDXoo60jjRGpNl0cCVRqT93bJtdB8xaBT+NQ7MyYIzds3I6kMQ5ppdJ+zzlWReSfBH1tsIcrH75GMo5KAYu+Mp0ElgawCtIXQTGPoH5GIo5WGyANPFiN3FMgcKNQ4UKuzNFZmO81SBUs6B44WSA4O6bdhqwtRwwHo6ZDMZMtFrUtpco7eZskBgMasN1GYKW4PRdAeqs9rEGnlaQ/W7rq3hdIcaYZ+Q7hXHEOdcefO+E9IcJHHEMIpI0tSN3SG0B3CpC9X+aM7BHCF0XWvmOMmOqavNiGjMuZj66bgNQVY5Cr4eBTG0/A0jfP/7Gw8SlyM0SqA8hNIg7Idy7GiP0DjU2LyVktWP7Cd7ztWuhdF6acxk9xM71hhz/Gr33y4T5HqBsmy5xq1HV/td5dQ+HpnzZQ/a8tjvmp/2mucr440AyexY0DPGgZACfWJG20BrjXQ56Zx2nGQZ1ddaDztGNVcF6t9M2Y6OdD1A6K3QkV7SYNBLSTRgC7gNcNX/fwbniTnHlQlHNQjuJrwpaBq34b0G5/mpAHsLMBnBfYed52DzGeedniAgxTM4T3Hir7s4hNcBf0JgCBz15x7AeZiV22aWYHzbSRfjwKcpX5/b/T1Wcd7vvbgNfgpnDEV5KORhug6VOgx6DhBZi+F3n3DnFfx1AxxwsqcO80dg/1FId5eZePePkpu7m9LcncRRTLJxhmj2GGydhdo+SFNi+kSlOlEMuTy88cv/lVd98fd59jef4dVLw8tMhvPAZ30b7fd1yvs6LPs+2Wf6KiXkC4pwLIuDvh3EbGr5dprC5Y2Z9/c84J/zXTEUE7h9DqYOQ/0wJEXIl3PE3SHFFNrlmGT3XqIohUvnSHvw8F/CXZfgSB4qA/hyDg5FcC4NTBSNF5vrZcWXu4rzGK4xmp8gIry9SW9bmongb8zCcMn9PoNjfxwA7iHkH6rhFu29MUxFbgzu3e+YWo0mFAqwvgbPduAtR6BccgNqcg5yk9A/Bx9/GGbbrk0/7ttM7KMF4Jv9/5cIwFODwMiY9WX6z1wJDvQIYJyYWmLeTRAMATG3RLVXkuddBOUSRpUugTtiUGHaUzmjKqa9D/tyHwSeYpRaX/bnTfvxskbIeyDlaJUQQlXy9d4ibGJl3ydruPmz4c+RYhz537ThWi9M4p/bJShiAiikmFSA23Bsnk1/zgRuvM8TEoCDWxNmcP13J27ciUF23v825dv8nD9fxtIdOCakgECtVwVf/v3A/ACeSsMb/AQ6rfnzGrixB4GN1MHNT83Rqi9/iuvns4T8Wgq3sx6nIYElh293jaEFArNK7Tbp+2fe1+vNs3BuAO1V9/wjBOCs7tv2rG/f/YyOS80HhRKeJOT6avu+2OX7RfvMboKxmwW0b7TcaDDo3/7bf8uP/uiPXvZk/fqv/zof+tCH+K3f+i3+8T/+x2OviaLopiU83pHxIpDzeq6zOc/8NnKZoaO8ZwpXzROABrFOZJRr/rYI68c6YS+wz5MTR84zAeACveW0sM4XAUvaT8WCnMKtY7OEN3NN+r+1MlQmoFxxbObqBFQnIe+T4rUbsL4CG6vQ2oJ+D6LYnTuzC3YdhtJteQqH76Nw6K0U97yG/Owx8pU54igmitz5UeQ+aQLDQcKwvcqws8aws8HgzCfZ+m//iT/98y6bg6Dr1UwbyBmkdbnlf5fOkw2jzzoQLcBW8e0wZdpnDreOzUUwOQm1aZiYgslZKMzgcwQUSPMVGPY8qgWdi3D2OJw+DyuJW+8bOJ35hO/TVa5kEEgHkhEjh6AdoxZgFLijfpUOoDVajs/dOL1gv/87n4epGajVXT+XPEML3x+FIuR97G+/Da1NaG9BawNWluCZIXzZP0tMsSIBSJOuIOemFdv24+TFAAM0XzTHNBeUSFwi/co6J6xTRuNGAKPq1SY45SygkwWEbaSEfaYABzm4r2UN0viF0UTdVxMLVlvwWfWRM3iCAAbrOn1UP7ENLSPdlsOCZDomXUJjQKzHceVWm8Nz59S6XikQ3oxrAR2VxeqXmkOqg9jhdQKzT+zDHq5OLUbf0D2uP8eBh2JOqZ2kAy4TdPibJTcaDLoVOtIOGGRkHEKpDp/L+41w6KmffnPeBRS9Z6DuV5GBt+yTISQJzKcQVSMaScpMHNGNcxzOlZjPFajlSuwvRtRjuPsgLKYp+5dbbJWq1NKUcq9Br9elngyJ45gkX2BXMqTXGzDfc5twPnLPUR6bOrCZc5NvNnae+DQfUczlaMd5p1kQkYtiDsQ5JqOYYpqwj4hD/vo5YH8UMwHMMaRSgCiXUMwlTExDbQoGvZRc3ilje2lS921x0K9+hyLYVYYDdZifA/YU4Ohe2HMEdt/ptJqVAey5w+32U0dcDwx7UJ2/3COF5V3ULs2xa9clpvptyknCgIRhmjLbG5ADpvoOqJrIlZlsd5iIfOx96havauTYLtXYb2gJ5GpFZts9SvUauX6PYrFMM8rRyReYjXPsxhnmMc74S4HbvUZ8aB/sOQjTt0FUAsp56Aw8dSJ2qApAbUDag4XJNba2uhwqQaELXyWiGsPc0HnHyv3RUJkcMBF5T0Hq/gr0yQPTsfM+VSKXWLucwO4irPVg90SJg/un2Z+L6AAzaUI06LI3X+JIFNPEGfQTQCFK2Rs7o3cqgoMHIvIFaDQSSsWItdKQ7labOw/UqJRjclWY2gVxHdYYsvcCtDox7RTm+i0qgzad4YBSlCMiZSbJ0+wNuDRILisLqoMFLmyerOz8k/IOYRO2lNzInKu/Nu+BPAm5zPVS+LVhazGUYiJDQYyfLUaTE2N+072sMiFwQXloBM5AUMDkCRJzq4RbXxTyppBHKRFFc63qJoaX2EXaNAW0iBU0wZVeYQhvANTmP4EbHw0Ca2zS/NWGrE1ZRt0Ebj1S8mWrXKo8RRzjJ0pD+1lmU8Ffm4tCO1kFUG2vdlYYFoz2L+Y8CEaB2kz3UuJVsSml8CrsTwyE2VKO1SilTHIZ3IkJBus0bh2U4ih2j/q8QlCONRZUPvWz5oJAT5X3Zm/S4zyBz3c+wObm5sjxca8o7/V6PPTQQ/z0T//05WNxHPOe97yHz372s1d9RqPR4MiRIyRJwutf/3r+5b/8l7ziFa/YRil35IWK9fZLNA/zjK7pURz0qVzk9qlq7PbhUhTA5Th186uZQjv17MEYJnJQzxeoFcpUcmUKce7y/B7i1qZSmpLvt6j2W7SGQwZ+4MapY/FVUiinTmcbpob951k8SQ6SCNLYsV4vG2sRxFGOShxTi3JMETEdRcxGMXNRzFQUj7yqfbKSUJlIKVcTKh4IqtbdXkmS0F0fsJ702OintBIYFFz7VCZhZjbHrt15CvsrcGQWjuyF/Ydg7jao7fa6GlyeZamv5LAHrTJ0qtCfgfI56of3sm9+nb2NLXqDhGQItdTpBrkkePLVhjCay88y2fPmXPVnoQC5QkwuX6KaK1CPYibTlElS6mnKXJxjd5xnPpdjajpiYgYmZxy7uziHVzpKUPBgUKcNmyndQkppq0ex0WSm12MrdcB9LoVWH5qDAIxbxq32be0x0h+sTqBztb/WgOkCTFXLlEqTEOdopSntdEguTV3KgCjmQBRzMIo4AOwuwPQsTEx7MKjinJSoXYqQ8xvCoJnQWk9obqQ0KjARpbR6Ay4NezSGPXLDxKVySB2jemLg1EYZyVkWq+qp73b+ZY3RcWv2OKM6+5vmrUAXiWXVRuYcyyQTO8My5QS6WfaHnm/rFpnrtC9b9oz2RgFQQ0bHLozfq3Sd9k0bVpQ9d1xZrB6p+lhgWI4d+zzVUyCaBZUtwGYBr8T834JBEUGXs+PAPs+W8YWAQdl76dkC7gUG2XaXniIWIATgq0IA8qVXanxI51a9LXsqC65YgA3C/JbuKVLGBA5YutmyHR1pO/oR3DodaQcMMjKF24QUngFOackD/+ygwzLWF93Gmp/0cdxAXIFiCcqz7ntadjfaXICVS9Dr5Xntd9ZImhvkp2okU3upH3gzhfl7yO1/I6VihTwwUYWpfot/9Lk/pfW6v0nSbxN/6bdJn/gM6dIFujNT5G+/l+LqAksPnaD9CNxbgqQMX1h0zI+HgbuAn9gN+SJMTUFtEvK3lYh27aM7dyeU6hDnobaL8uR+8vkK/e4GtUKNahQzSBMKaUKpPEUuiilsXSAXA90N4u4auWKeXG5I2u8RpQOijYh7Pv17TKcpsxMwO+Xarl6Gmb1QP4RDKqZnIelDdxM2zsDmOVj6qivLhb+GXtNxi3tNmL/HuVsGXbhwnPiV38V3/X8eIH/qMXqtNdL+Fp3+gPf99TPkcwkf/yq862ielb3fwIX//pdsTsKrB3Ci6fr07RX4fAu+cxrmI1htwBvf92rO//eHefMHvpveE49ReP07GVbmSQ7cQ2Fqz4ixLuPNV41KEYpyPcZAHEHip32Mcx8BdDuQwsFn/3fu3Pwk5QlYXYUHNyocPJTyHWttzizCLz/mGANN3EJXB35k0imzv7zlwJrdpgx/Zxb+cBm+qQx3e1rFj7wa/ssj8A++9Y3c87/+C17Xj2kBhfYa3fOfp3j4HUwWa2GBTlPidEiRlCKpAw1qeaIoZqa5RlyeYG5tkb1PfojZN/9t4vIkUQ5iv9NOLl3knZvwpvJuElK+7Yn/Snz8v5Ncukhcm6FLApcO8tVHTvH5U6ts+HpN+3k1j2NlXMItmEdxbzqR8S8PZt3Xe5mwoch4bvn/KzdTC+e9vUTI1yPWxW5//Yp/dh23mewigCViIy0SknxPEjzY8/7caV++2/31Yu+dy9xrn79uDQewxjiW1CKOTSJWTMWfu9u3QxeXw+oVvk6nCInX9xAShtb837sIVPQ6js2z7n9r4xg+87i5UCAwomL//SwO8NSGe9C33Ywvy5xvtzpuPvT834o/Z9OXSx7j84RcWCdwObIa/jhNODR09xv441IG9vr2mRw4L7FyXEDo35pvv1lfH+udEqBlgS4pElJKpgjsqnt9u6T+ejGoqr7N1HeVqX3sb/cYLF4iJeSCmPJ1z/u+O0WgSdeAV/nzxHz4pG8nlWe3f+5hnCf8GGGcrPp75HHz5GbJ9YJBhw4dGjn+Mz/zM/zsz/7syLHl5WWGwyF79uwZOb5nzx6efPLJsfe/++67+a3f+i1e/epXs7Gxwb/+1/+aBx54gMcff3zkNeo7cmNlisB6lHJfxK0Jh3Hz5SiwuwSVmscxUgci1CZhYhKqVee0ycXe8BxCrwedHvT7DiiuTkLlYET+6N3kbv8m4kMPENf2QBSNhBn0+216T/8Z/Sf/hGTxLOkWpB0YtJzakPScKrG14nKU9XFh7dNTML3bARX5SS7TRNK8C2WiUITqHLnqLnLVOfL5Cvl8mUJ5ikJllnxxknycJ09KPh2SG7bIDTeJh03iaEiumCeuVB19pLdF4dJxZs4/zuSlDZKmUxOiIsT1PIX5feT2HYHdB2HmsNPN2quw9ASsPescZmkK6dBTgvrQ3YDWsqtcoQL73wCv/D7Ku97Bu9/4UV716V+ldXyZ5jJ0t6CzCZvrsDkML2to4NatGqPhqTL0ar5fZ3Fr1IE5OHgvFO6dIbrtAXLz95IrTZIf9sn1tsgnQwqVWYrTt1GcOkSumCdfcKBJXhZ9HocMxjlXlySBARQ6Q/Y98xAzX/oD+mcfZtDoO0ZBE86egIePw1cSt7Yu4vY16yyRg2aSYIDrI2OzhFvn7wO+4e6IPd/7APFbfhyq8yTJgGFjgai7ST5XpFicoFyeppQvO1Za5IGwvCt6nHOOYEmsh0WQazeobS5R7nSZGsbsave4fe0U33Dhi7QuPkJvdY2kB9EQck3YPA9PHYdHh0732SIwRsTM6fk+2mI09YMFPCywoDoL1IsYNcAx89cCuJZ5lxLYz3LWKIxJ4AamDHHmmPRlgSMCC6xzRuUQCKPyiVkTcSUAkDP3Eeskm5dGLDg5zpLMfS3IlQXYBAjrXOkOcpDVCTahBX/EerFsHwFJRfMMyxqy5ZFDSrqj+kdzU9foPi8UBIIrw0f1XXNKTj4BWZZhZceWZTeVzXUC4TQn9RxFmwg0s+PVjhErdm0SaDbpy5kQGIQ3S64HDLoW/QhunY60AwYZmc651x5Xc84bEecjJpOUYi7m/vkqh0qw2HP78uS0o/kSOcWnWIbaXsh5Xm66CmvpkIVOj7PNmPtvKxFvxjBfgrkpuOM22PcKOPY2KE64Angm0FT3SXjHO6DfhOQT0H+aXq1BumuG0isOwmLCxYVLnDoJByswqMJa0xkZZ4H9MbxhHkol55WpTUPhzgmi/btg7xEoz0CuAJP7Yfqoe35rGcpTrkJp4kCbyqz7/4rPMtNcIm2UIS6QDgdEwy7poE++FFGNImppylTeOX8SIFdx4FhSgE4cQ5InanagsEkyLJKsniVdPkeuvsBw+TxxYQ/EedJegyg/T5yvkAw6FNZWiQ6+hkOvmCGqN2CrBv010m6ffRcWyRcGHF+EV+4rcvrIYTr1GrunI+b6sBY52vLRcpfjcYlXzMCeCC7m4K137ufRzz/LG197B+3eCtXX3UtU2we3vx7mDmWHx3VLlKbU5ufYU4J0EtoRHEhz3L4fbivFVHuwm4Q1wsI5X8pz73SZZNBjojdgolBlOh0ymabMxHnuqneotYscmYg4UO5ycFDmdftSPvvMgNccO8jcm9/B/jh3ud840YS73g6lyVCwNIVk4BVOvz3mSu57awlKU1TWT1GPvgSvfpMfH16SIYXp0+zNFy8zuo6VH4XCQ1BvuYGXJpys7eP88YsjSog8CjlCAmQBYGLRiFXRIrAqBGJZY19gkDZngUjapKzHRR4dbUwCecT+kGexSNh87KYnxUAKhzYkmzNBSaTlGav6v21Crqh18398vRXSWcABTQkOHJwhKAbaTKf8Oan/G+EAKT97KRJAHMkco4k+JwkAg2jPUrpsiIU2aNVHiqUUH9v2lqGj0I8hDuDUZt8C1gYunExhWlIKcqbtColjvlmlR8pXbM7X//W7nq/xpRCR2Byz9HGBdl2CUqIQlhkcYDUEyJWo5Nxv8loJlNwkjGHLXMr5flGIXMe0s1gW6nvVewbXp2IRzRDArpsl26FA63yAs2fPUq/XLx8f5/W6HnnrW9868hr0Bx54gHvvvZff+I3f4F/8i3/xojxjR55fJgjshSIul005cuzjg3GOO+M898Z5DlYiJqYcGJSmLtfNxJQDXyoTTvWI/CRMB9DrQq/ZZ9DukQxSatNQOhgR3TUFr7oD7nqzi8nO5kDoNaH+LMSfgYl1WE9Jm9BtQq8NvY4L2VkZ+gT2qWMnzc3C/B4HCJWmIaoTKIL5yClzk/MwuQ9qe5x+VKhCdc6xdUpTDhFIBg516m6QNlehs0k6HECuCIUaaZwnba+TdpeItvIUepB671JUwsVlz5QYTExDcRdpWiPtDEmHG9BsQRSTJs68j5KBe96wD+0VaCzAoENUniFXv4/87nlyd+xnX7rEvqVZhr0OmwXYKkMj79pgaeD2nk1G18IKo4mbtT9M49avgxEcm4VjhyKKd8/CPUdh/yugPO0ZPuuubLXdMH8vzB5zbXAtkqbEwx6VwhaVjTrJIIJN19VRE+qr0C5Cow/9NORb6uBAmnrsmNH5KCKNHNCUi3LkophcFF9e08s4/fiOCF63N2b/qw7B297q+njYd87Jzpord3kaKnOuzyHoR9KRfLmv+J6mRJ0N4s0L5AdtSrkCNRJmVmtwchGmTsPiALop9CHZgku9AYNzXdY7Kb3EMbmk63T9R6HEAhggGO9i+lrwSOu3ZWNYMEZMMF2fN/e1xrj2fgsGaY+zzB3L+JCOVDS/F8w9xu3VOXO96t0312s/FrtYOmCfEOInXU/1ki6jslrQxgJCWWDInienpNW9bJtZhpqcSeq7ceFdifk9C05h6iD9q0pwvFlAyOpDAtJSRnWl5xO1rQ3ZVbvp/9Kb9SyVVW1lgSI7ZlSXZEwdLfiXzxyPzW8SywyyzGnp3irrzZbt6Eg3Wj+CF0dH2gGDjPx/H4AzX4Hb7oZzX4X5V8zSvdSgfs9B7vhff45qPkfFBzrmiz5uGOMxkLWZA7oJEye+zP6H/pzKZ79MNLXXucWmdsHUIagfhMqMY8VYGXTg8P1Ok8pX4NX/dzj4LvLtJmm5CPVZ6LSYfvUid/2dHBQKpHmYablJ8SqcUbiv5jwWhZL3zNTzUKm6jS7nl+pCBQo1V4Zh11E+0qErw9ZFx+BZOwHP/g8YtB2lZXGd5nJMMnSbY2MzoTEBf5km7EnhSAMme86LE+dh7yWoPgO9So988SL5P/tdokKN1XyR870GG90t7qh9iWfbK8yVP0Y5ilhLBtRLUxyJc5xPh3xT/ylqHznOHR/4Scpv/HvQ926/JKX6mnXiKOWb1mByMuZoZR/zb38br5qoU0zh7X1opCnzH/0N3vGOv8vt9QnKacLdg5iZO3fzmvt/kOi1d1I6/G7YdcAF+tdmXtyBNeiQK65z4ovQKUC7DfVim80De5g6VmG6UOWuT57i1RH8dQoXcvBPvu/1vP59/xCe+EP+1TMPkX/wn1FfP817t85TOfpu7nr4l9h/2/ewKypQeeQ/8dp3/W/srSzx/W99iIlv/yFDL8d5Gw+9zScxyEgUO+Cv34LWihuTayccgFSqw8UvwROfgiPv8u7cvDt/5WmXWOgV73WAIikc+2bYdR+0W25SRBG7NlJqX/oZ7uDi5bwLYmbM+PH62gqsd2AydayQaRxLpQx8DPgG4Fkc0DkgeARkUM/iNp1NHOiheOcCblNq+XsKZJrzfydwyoQADXm8BTAt+9+rBPDmlC/DBj5EFGfoT+E8jzUcw2PJ10EMkZY/J8EppRf8fWqExM55U+8Ux+6Z9uUc+Pp1cN73ginTNA5U2OvbQvHT8piu+fudJ7xp5iTBsDvv6/4kIdxs3X+e8OW6zf9ucxt0cR5mgTypf/6i7yt5BBN/rRSmSQIAkuBYBWUcODXvyxt3HBh00JdPIFTLlOF2f79F30aJf+YduLF1yf++2/eX8jOUCG9VPO3/dgk5k9oEAPAizpB8amGR6f6QJwh5okrmmWdx426JwHBqEsblKo65edg/o0YAhy4SWE5DwtsbE98e12hWvWhyvcyger0+ouyMk/n5eXK5HIuLiyPHFxcXrznevVAo8LrXvY7jx49vo5Q78kLlXbh1aBo4NOGimep7oDiXY3L/fUwfeoD6wbdQqdbJF4JxEsdOX8oXvLqToQjk+33iZz9G+tU/Jb14gXyEC7vOFcYVI0i+5Pac2Tug1bhseRQGLjSsNHT5+SY7sL/njacISkUoVZ1uNEIZVLninNsrCxXnHInz7pMrut86G7B1HjbOwvKTsPg4XNygv9Kn00jp92NS8gwGEZ2NHmtLG6y0ttjqQn/ojfQY8oUBueICabFJt/gk7VyJZlygHefoRhF9IhJSojQlT+qZvAnDYY/BoEOUDpmKi9xd/zJveOCDTH77t8Nt3wDf/ovE72hT60CpB1M92NWFo+noq+QtS0Fz2IIIlikwUYHcDDBbdgBKZca1RzJ0+kCauv4Q6/xaJU3g0qPw6B/Q/dyjXHp0QNKF6V3OnzR9G9xfg4ObcGkLljdhq+NC+SanYdc+KO6CqD4BE3th+gjR/L1Es3cQ1fZcHkMC4WeAmTng2EGnC+P7e2KPq1MUOz047w21QQc2TjsKT2sZeg13bNhzOnG/7XTn7iY0VmGl7QpYHMKhvXDvd8HBNzum+yv+JnS7lzfFqNtj+qnP89pP/D67H77IiZOwvulAS+2vCh/bxO3HPUZZFvhjDf9pMT7XjECOLDPIgjN2DMjZIaea9u6B+StQSR9do33RAkRyzMiRY/MRWXaSxh8EoAVzXGFIcthZZ5Ktl2X1q/4d3N5u28iCQZbp0zVlTHF7+Sohf5kAtn7mfHuNwAuBHjpnaJ6nvTYmgFpyuU7i9AK1S5K5Vowxy6TSbxYYsv2qtqkQQtwF1gl47BDerqq+0VhQHwsISgljVA41icCgLFCXN/eyjrucua9tG/1fx1QGO2ZvtlwPM+ha9CO4dTrSDhhk5JuOFDj5bMq9ByNOnIYDh+usJ31m75xn7t3fCbni5TcxjZfgKYiSPoV6hYmtLxI9kZLkax4ynyLN14lyE5DmiQZDSHuQesdXrwMTu4lI3aa651Ww51XEGa9Y5fAWlVzhsoE/74/fftWiBe9F+OvKbL6SDvuQ5knTHHSapOvnSRefcHzjpTU416B5xjmCogjWVqB7EI6nbkGI+u4V8I/61jiwIWM6IU+DIo8S44zhp3BG2Rt4hi/hjCUxFuZwm99TwH0HoF5c4LakAPtfh3OtOcp48Q5X9GO+CqVhn9n7jvmcQ75aSUJ3+SOUv+s7nAKQDi8rLJX73Dn5fWPaKzUE0pHvqfNCJenlxWnkUjW3/yHttoE2SxegETmKcG12QLtdYnqqQGmqzgxwewTnU9iM4R2vPsD0t303TH6VA9Uz8N3fBYtf4Q0rT8Nr3wfDP+G2t7+H4aDEsPsJit/9PbB2htdNpvDKt4zWIc45BQ5g2AtDQHXp96DbgsYKaZojXT1D1FqG0iTp0jOwcBpWzwB9pwAOOnD+89CG+Ng7iId9Nxjqh2HmdiIBTFGOWnuVYnXicrL0Kk4hy+OTMwK3552R3COwUQ4TaL5HcEY2uA1AmKvNsaNNrE5QUrRp9f0zpSDJ41LEj1lGPWBSUlqEzaxPAEmkhOCvFdBT9X+X/b2mCJudvDBiDsnTJxtEw6VO2PgmCcq45+iREt5EOCCAWnkCQ+mcqUue4JkRmGQ9ZCkhofSSr6fK2cclshd1+SIh+aWAjC4OvFHYmP6/Ydon8vee8M+z+XlyhNwbliETDUJ7XDJ1l0cth+vTSdxaMkHwis7igLohTqnZ468VE6nq+6jl+3ON8PpncIqfQEG12fJWg3LiyjIgULq7BFByxl/bICiIfX+s6dttGjdWC5nnCeQSI0hjQqDTzZTrZQZdixSLRe6//34+8pGP8N3f/d3u+iThIx/5CO9///uv6R7D4ZBHH32Ub/3Wb93Gk3fkhcodcUQ+jtlPxD3ViNv3wsxtEB0swJ374L7Xwz3f6vbekbecZdRm81tEStzvEhVWSdc+Stp06tAwjp3FPxhCv0/Ud6uykigTRURxHmaOOTDo8v1G2X8C468qI8rPlbrR5X1S1Rh2SQcDGAwdG2jlJOn5R0lPb9JbgOaaU+GSoUsUvbUKCytwLnEswy6WaZCSo0FCgw7B4N9i1GAVM1OYldgTMd4hER/nvjRP9fVHiO/9XrjzCFEUU4yiGwMkj2snK4mHC9LML2nmawokA9L1C6QXHqd7dpG1UzAcQjHn8lIWp2BPEWbKsD8PG0PoRg6zmZmN2HM4Jn8khvmay8a95w44/CbYdz/MHB3vALP1EOMnV3If6cPDhJTE6W6bi6QrJxz411lzjLR+C3pbDhzqNR1QtH4BFvqXkwtGye1ER95JVN0LM8fcmmr0+KjfplTtsXfpQ9TOQ3ze7UVd8+ngxsIWAQwSsCUjXvvYBuGlBNoLZTD3CQAGBN1ARq1ls0gf0nkyxsV8sQwky5KWU03XaQ/TPa1hLxBFz5aOlDPnX+4mcx/pUxXzfzn9IOh8CqNTGVTXIaO5OQWu2Dpa5o4NFxOgobpZMGgcuKqyST8YB9JZEYNNfVsk5N7JgiRaA5rm+WozgS+2Xmobtd0kPvk9QY/bNPVoEfRL2zcWsBFA1ScAQrrehtul5tliIKkdS+Y3K+P6xjKoskDezZbrYQZdq9wqHWkHDDJS/Fv/B0e/qUBlX55D3wm1fTVKJ79M6Y574ak/hzu/2QEJw76nkKaeC504dKS55DaJladJFx9j8MUvcOpjj3PxTMLSfzlBPx1QKKzSLJ7lyOQZeqU6u2vzVPIxw4F/A0XSYuq+PUz8Lz/hDO/q/HhPS3fDsXqea7Ozkgxg8yxcegzWT7vNq7sFzSZsQWMZWuvQ6iUsrPbZ6G6y1W9zsrXEhbWLVId98u0uG03YaEAu8Ul1O5BrwqnUGViLuMVp1T/2OMHrrRCLKg7wWcFtdo/4a07ijMIhIUZ6E+h04XRrwKs/89+Y2LMLDrzx6jTkOBfC7iRRRP7t3xdov5Y1czXprMPil93fNIVLj8PyBShX4VIDKnkuPdFndRn27HNYSjJ0j+924KGWC9PbTKDZ7TP7lZP814YDg+6egAM9OPbQMnvOpHQKW3wc2Je4BfLOGPL773Rc+nwR3vC3XX1njvrQPeDoO6C+n6jTIvc6P+GrM3D0bW48Nhdday9+GRYegZVT7l7dLr1L0LzkiF5LbdhIBrSHPdq9Bq1ClTOtJSYHHaK4QK+xyPLCJtVn/jOQMIhyDJMBjeYlCgN4+0f/Ld/6yj1Uj87SPrtM6VCBfDFyNPtuA3ol5mYG5CZhqQn5JCi2W2rroTOkT+KUmtsIIWEloBnDVhLYM9qAIn/eur9NwX8Xs8du/A1Czhl5VMq4MQhhA9yN22DtxquxO4MDRAq48X0XjrVyiRDO1cSN7QEOpMgT3rB2BjeexUDZ8M9cJnj39vh7nCGwoCb98y75NnvGl+ssbhM9gAMZTvv2Ou3v28Jt+Bs4ZtWmf4aUSIWxbfn6NQnKlBg/PZxCIsDEKo362zJ1Eshkw/a0oRcJYNOKr6OUjrbpK4WWSYGVcioFqchoWNlduL7cZDS546T5nvg2XMMBzUsE1pTGxAYh/G4S17cNnMJ0rgfracjZIPBLilHRnyuywbov/xcJ40XlP0UAiJ71v4lJdcb/nfP3/xKOZXQzZTteL52/HfnABz7AD/zAD/CGN7yBN73pTfziL/4izWbz8pszvv/7v58DBw7wcz/3cwD883/+z3nLW97CHXfcwfr6Oj//8z/P6dOn+ZEf+ZFtPnlHXoi88x88SHXXUWpzdzNdK1OdxcfJxjC9z4Vy9Vsu30171bElulvOeZD03QY57Lpz+i3322aH9rkhz37haU6cW2RhE1oJpOU+ufpTlKf/gMr0F6gVJ9ybnmK47QjU33wE7v9+mDzwAmuVwtYFpxM1LkJj0e2d7TVo9aAFw4YLPet2oNca0lppsLGxxmprheXmIqtbbTYb0GhBuws9bz0miQOGtjwQ1CCsadaohGBkW6BexpAFuLTndQm61KUEFp8Y8sZf+5+86alNJh/8Zrjnu5+fWXXdTZbA+ilYfcbpR/22Y8kMPcO4vQrNLZcVvON+6nUdONbvOl2p03KvlO+kCWub5zh37ixnz8LFdcilsO8UHFlxqSYnB5DvOrZoLnHhhtOHoPLqo8QP/E04dBuUy46VVJ1zDKHqrudnKKVDuPBFOP0Jl79y2HdJq9Zh/SxcuACL/T4rrVW2uuu0ew0Ggy5J0idOhsTDHrmkT244IB50SLpDem0HZtYKsO/JZY596Xc5dP8XqBwpwtxuzz7KuTnRWoITX6Lx5UucegZONmA5HWX4aBzI8JcTRaHwPQKIuOHPsddZcMMywMTgUciVZZ1YMNWCIknmvjDKIIoIOpucZTpXv4lpJMAgNffQx+4/BfNRCGOVwPoRKIC53t4Pgi4nvU+OF1uG2JxnwYzE/M0yptIx56Tmo3onjIIZVxNd0zTni8Gj1AoF89cyrVKuBNMsVGv7XboUBOBwC7dGrRKcWWoXOx50DwE7OX9uk5DfSPVUfXSN2s+G6VnwLgsg2r9qf42BMiGh9/L45ryhsh0d6XrAqluhI+2AQUby7/ghZmtlyJfYBZCmVE/uhfl7SD/3q3D0G9yGMWj7mPHEbbiKHW8sOCXi3Ofg6Q8z+PxjLH4WThThiccXaaduUq8A9/OUS+waQb3oNsnZeSgWofTeV1J7rwcvytMevMgAGN2GD+u6xqGWDJyic/6LLlnz2kkHXq2swhK0TsD6eReu89QFZ9AuAZ8DHiO8HnoBt3DI49YGKhtuEVnBGVdr/pFakPbgJuwkbnGY8feRYtTAMURWCcZnE2d453CJJc+0hvSe/ALp8ttg7+tc3c1zsP/LlUK7RBFRFJG/7x2j55h2G9uCvYYDgLYuOCX2+IfhxBNQn4anl0imS6z/ZZfzJ6B2LzQ2YNB30VGNLfjcOkzUYGEAq114ZRX+rOuNwCkXv8/ZTTaOQ+4gPE7w9u2LInJz+y7HwHPsnc4LWtvjPgB77oPqDDEDuP0N7lhxAvbcQ5omLuQriuHiw/DkH8Ppv4ZiFRpN+sdh6xlYOAPPbjrQYoMQzvKI7w+BGyeA2Uc+CYRNdBnPcCkc591vgvKbD9H9ylkKrymQVmLYewdsXYLhXiYnasRV2GwDiTNuh7hNuYzTLWtp8C5UCZtrAehEbpxZj4elLDcIG+QqTjESw0YbT5tgwMe+rSdwY7ZGYMgIuBFgkhJy31QIIWdb/pkCafr++BLhNbir/lk1gtK2TDDuBYRI2V/x/bCGC6tS/PZu/7xLvj8u+HI94+/f9M++5J91yT9PIIfmUp/AnpISJGBG3iFt+vI+2jkqBUGeNm36XUKSbymeAkkkCYGlVPDtoHJqJufMuWLJROYjb1+dwJ4p4wBkUY3F+oIAPuPLIoWn6Nu57usl5Vl9i7/PCX98ClgehH6VAialSnXaIMTZN/1xgeFr/p77CcCavLpN3x4D3yYtgvIrVtTNlBvJDAL4W3/rb7G0tMQ//af/lIWFBV772tfy4Q9/+HLCxDNnzhDHYb9bW1vjR3/0R1lYWGBmZob777+fz3zmM9x3333bfPKOvBB51XtfT/3Ym+G2b3SGN1zeR1NwOtD6KVg/4xxPjYvQXIbeJvR9aE2/6XSkzjo0LsGlLbqPwunPwBe6jgm8CqQMKXKBCS4wzaeZxekRd+Zg9/0wWbsf7v02BwZdqw40TrRXLj/lQKyVp9z3zXOw3oY1SJahswrNTWhuwPoSXGw50P0kDshdxM1/AeYQ9ioZ8nYPs4br8821KHOeQlkg7BuNsyml81/hVZ2vMHnfJOld334ZDBm5fzbv0nO2jenbkeOJ03UvPuzCp7obDvjrt12/bpxxb05ZTUk3YbDp1NVOC9oN2Fpz+tJG163J53CvXn8Ctw6WgX1tuPOSC/ndRcjNN1OFuX0wfQBy9+2Bt3436YE3PX8dx9UlGcLy004/WviKA2j6Q5Lz0HgYzj4OTw1cP1/C9W+XUSNWxrkYW23/jBngPjYpf+UT7F78BJXX5kiP3unSQ8QFNyc2zsLZc7RPJlw6Dws995xNXLvYZ4lZor1X39sEdqucSwIztBcrNGmkiRhNEpyYc2ScC2yx4U1ielxNtMdL7+qb7+PCndQnAoVsH0UEvU4gQW7MsWyZrVhwRgyWDmHPt9dHmfNVBnse5jhjjmefbcO3nm+Vkl5ryygdTPqn1g2NOV0jXUggWRZUs85N+1c69xrBhmsR9nS79lgWjsAY6T8twtjHXGufb8Egywwjc419ri07jCYUFxN/M9uQN0FuJDMIbo2OtAMGZUVJd8FtepceI138Kv/tDz/J1kd71NKEYTKAfJktUvZGMStpQisZstVZIx20aa2fpre6yOAUXPRW6LPOSXI5T+ECbsJ+MYXiwIe7tOBYFw5+dZFnfv7XqecKVMrT3Bbn2CC67NXfDXTbK7zhWJmJ+RobF6G2D3K5mP5GQjqEjRWXM3iQ+hwnyYAnG4vUVo/D1gWWvUen2Ib5Jpxdg9aWc5aIzryO+y6vuTzyMc6QhpC7pEzYIKv++gP+OhmdESGHRo1R1DwmUD/b5poc0EugnIMvPbbIo7/2x0ztfpq78gUeuA0qd+2Fw6+DiX0OpGtegsNvd96X5iUXC66+jCLnjUz6Tpk982noNemfu8jG42dYS+CjK56x1N1kduVpir0tNtOUeHWBwvqQSrHJ5CoslQbEyzCZwCcWYakDpQQKMQx6bmFt9WDDOZo43Q8L32AAe/NwKYIvpzCz4drkwzijfW6Q8viHP8zrOqsc/4vPcOxbNsh/yz91bK582Xm/mktOsR72XI6f+bvhod+FfpvGeo//+JHH3Sa+8gyd5XPkNlLyuf9/e38eZdlV3vfDn3PHGm7N1V3VLXVLLaEBkARCEwLnxTb6QTAZcLKchEWITPySZQIEQpYTcGxI3rxYZGUlHmL/TGD9cJwVE2xsB2MMOFgMLxihEc0zatGtVndXd83zHc5+/9jnW/u5p291V7W6qrvV+7vWXffeM+yz5/083+d59qlTqENhCo5Pwcxq2MxYE6pCVg7QHsqyRCA1JBAkwMEW/N4BKDdnqU1A10KL5VJKszbBSn2R0fQolbkyu4vwtSS4OS/gFe+LgZ1NT2wczdp8Kfst194HUy8wihDQQimlWV4sIqpEGsjCUsIr/pMEoSDN+reIiHp2XSO7tknwECpkdTRNEMJ7CZ4zEoCmTdoQwonk9ZPSbjUr4IU+Rxg/hwnWGllKRP4o9EnkjRZMjU9rUdK+NxIGrceOFnD5FOpYX1avCuOSEKf4cUcgxCqE/XckjKwQFnuycxrLRYJgOEOYP1Qf03gBWt6Aqq+jhNA0uYOXs2PP4efRXVmeJrLfU6Y+5Yk0Q7C2zhIWPs1B+l3I8iFPMnu+aa6RIqeySkjXsXF8u/UQQsSUpsLLagSBaNGkoSW+QXiD03ZiM1YvXb9ZfOADH1jX5fnb3/522/9f//Vf59d//ddP4ykRZxR7/waM7MvCaTKkTdh/J62Hf8CLj9c5OD3DxMo0c6tzrNYXaDSWaLRWaaRNGmmLZqtB2lwhaa5QaKziFjwndLDpSRUZhCB40FQIG/Q/mMIDL8DYX7xI7/7fodQ7doJQLi+EPvwaM5RAXw/09mcbWJdgdRmmJuDYouP40gSz84dZWDrO4tIkK8uT1FcbpCvgViBd8htSp6v+s9rwY3kKT16IxLeWcavIyOvHWsylWG1EoUhy31a5kEI3Azzq4Is/gp3/4/uMfff/wyXdJXbvg8q+Edj1Ghi7zssOG0VzBY4+DM9+l9nHZ/nRYU9+HXMpy/OHcLMHKK/OU2muUmqtUkh92zZXFmgsO1rZdjqNevZpeNmovgKNZiAYZvDekofx86DkvoTgUSkFuK8OFx2Fyx6ES6YOsu/QZxi98Wskl74Sdr3ObzLUySMq8wJqPXAnzz24wuMNOJi2mDv2GM3DByks1CmlKaUU3DxMH4GjqTfuzOLb2xJ9ncKR1BYpoR8v1eHJ56C6mOIeP0ZaWcUVitBcpbg6S2XOsXAI9rd83R6nfR+chBAmvGKOSUsRwSHPWhsedjKPFEfok/l+Zb1BROhI5tvIXG89ZUQMWYWe3O+8J4g9butbZJWMViIb8h47+XTtWJQ8pHGaJy9s3s6kEWYj9Zb3jlI+63i5qEzYaqFqrpF8ZYm9Tu2udrFknQg+ydN5gkb1qt861yCQ22rnTu1nf2+UEDvZMZVLz5cusd3YjIx0uqaK7ZaRIhmUR5Ingx6DY8/zF1/+a44e/h6jBGHlCPAa4Gn8JH4EPziOEzZUBb/h66HsvireOvxjvLCyiI+RHga6l+BNwCVPH+Pz3/9/2I1XTH4Sr5wfxU8ErwbmEv9yqJ49MP0AlG+AcqnE6sEmaQOOPAU/OAYrLoQffBmvUIK3wIEXsK7CW7BbhFcmS+kTiyvrdws/AHsJyvgcXuiyZFALTwbJeqGgLpEJu7NjmvD0TEfYIFZW9kYLqgV46PFj/Pr3/pw9/Dk/U4br/1/Q/bbXQOF22HV9eCXrxa+HYsG74baRQUVvwWoue1Llx9+FhQka9/yQyT/+Ps814f95xhMTs/j9l2pZ25XRfjdL7AKepMVVwI3A/5jw7Tlk8nwNsNzwdTQLuEbWvYBWE8YL8FgC33Gwd97X4V9l9TbahCe/8X94zfJf8eyftrh0xwuU/q+Pektr16AX6PRq2VbdW2PTJjz4h7A8w/z+Bf7bbz3MQhqUa+9e2aCEVzAnCCRcJeuHNYJXxyRhXxSytu4z7bVGBqXwzEGoH5zjCqC0P2UCWOEYs8BVLHEdcN0eT37NZ+1/KKvP1wLXNn3/m8iet4xvgzS75mHnBUaRQfO0h1WJjJSyLRdaCQ8lQqiVFhKHT1PjQSRVM7tWHiMQvGZmCd5FPQQPEd03TViMrcVG3jnq0xIYqybP8oI6kj1TXk96bi+eOJvJ2kbtIjJIi7nSWsG3syWD5GJeJnhlaTzL4rKYlc0KTPJyyZNBDQKhkRA8qlS/Im1EqEkImcFbfEWCidzro50Mcvg5b5VABol8A69ATOLJuxfx/aeWHZcFMSXMMyrTPO3CZJ4MGsQrpiKeLBkkYU33q/6gnQway/LTg2/Twey5LVPPVdr3p7KhIHpeH9v7Wnk4UXDfyPURFwD23Ar5DTDTBjz3V7T+/L9y6Mt1fjDlvVxfxI9jhWXIyCCLvMaQ1ns7BhUyYOdqodvB8Iuw88XD7PjLz9JFu9KmMdyLJ2QvBfYlsGsH7LgYymOQVP0LuQ4/Do9P+rXmAMEbeprgYZGaNDUHWUV8hbBf3WbHzJkYN45ABj3u4MDzMPr83VzH3SR9sOOnoPKTr4DXvdt7pWyWDHrxPvjebzH3pwd58EH4tvMePNPZs3vwc67knibtdaJ1LF83an8ZGCfx/cURNgeWV7JCnxKgpwkXTcCrJuD1j7xA7bHPMXK8DG/6Wf+20/6LOpNBaRMO3Uvrq/+JZ//nLH+x6L3eXyAYTrrxc3NK+9ukrCcFBDLGEkJS4NVnZIA41ICu58E970iZpMHkWttXCDLXcbxMNEW7F0+JsMegCJ8zNd+K5OkElUvXbbRvq55ENIh8WA+WYOgE62V3Kq+k9WDXaxFC+XTksYc5Zz1s8jjTRIA1MuXJIAt5xGje03YJpdyxBu11p7zUaZcH88+3BFkngk1o0b7PYt7zLA9LBm2EZMsTSYL1KtMzzxYZtJkxcT4gkkEGrb/+DKv1LhotOH4EFlot7nv0MVYXpnh60TGJV07UCRXKcAzfIbWYSdhpEd7IoI6jyXGQEO7Qj/f2WXN/XPbnJXQ8jScbJvHE0n1Af+L3+PnyMjw3DbWnoFpIWZ2BpSYsLcOTLsQZS7kqmW9ZvWV5LxIWsbq5VpOClOwK4Y1CVYJrooieYYJVr5uw+a7235DHyWr2X3lcpp3xJvvfaPhNA1eaPq8zwFwLvnwQur93jMKh7zI/sJ/x+hK7Fo7gHvi/OVooUVg4Qk/fLg4BlbTFclKguTqDazXY0zXA8nM/wNUXmH3+MD+e8nsVyJMJgtv3KkFBPp7V2XzWFx4neHpY75IjeGVQVhsJS2W8lWylBQXnSTFH2Nl/Krv+rklH+ZGU6SX4o7unaf7f/43K4lGqpW5KvTtIDt5Fa+dR5lurNI89hvtBk8oPnqfQWKY5ucqsC5YqCMTKCoEcEikhIkR9s0LYvFhkYH6xqAA34Pc/unfB5191N4Xv01IbngAubfpQsOXs+OEsrUmgWgO3CA0XiBgt/BJ+RC5oQZeAkkWerX1U/7LSqp/LuqZ+LqVkldDPRR4oZl+oEkKl0qz+NHbVt0UayEoySPD2EEnTZ/KtsS2ry8UE0kV9H/z4mSWMGVkG1Wb9BG+pHoIA0Jvda+eqkjlfJwgeEiIkUEgBUj5UN6sEockK+SqLvUZjuEQQClcIQqa1qGqetHNTb8Hv0dNyvs66CWFfIsmUtxmycNXsGb0EolNWLxFbNYKiKQ8d1ZWUjjnavaisxVTCUTehb+qYvMBEjM2bulSdzWftNEcgC6UUYe5X/kXGbSc2q6ieL8JOxEvD1B/8N2bSblou7BUx1aoz8cwPOfpEiydXPKl/mLB2KhxAskHec0F9xypdVsjOK4tSBmQtF5neNPcl+LE9hZfN9jsYWIK+CehfgXLJvwjz0HIwsk2Z9GQhVx7t+BUZlPcy2I4xkFdW8+dErB3Cz01TdXh8P/QXZ6g+fzfddzUZ6N3BCDBU9dsfOgcrizDXCK+dXws1aiyRvng/zSfnOXYMHnG+fY8RXrygdcDO35pz67SvE3klU+uG+odtf0sgyXtBmCCEek/Owd6HU6pLz5A88mWS4ccoFCttRH8KuLRJevD7rD66ymMNL1NP0b5eFcw99Vwe8nOinZ/tvXnlXXLtKiH8R2lKvtIa2UlRlqFGxi3JaFvd36yyezqEhsbMRp6zkfNK76WU+1TrWv6c9TjaDNF7OrBjQjJhp7xabx4rDzpznw3ry2O9/tOJ3FB9qT3z50/Wvp3ayrZh3iurE9Y7n/f+Ohvyx2aee77IR5EMMmh++VdY2d9ioQFP3uVdlz+VpszgaKZmw9sMfXgrWL5jiAhRnKcULAgCxE5C6MoO/GukC2SK56I/pgXwh3gLyQKeaPkh8LoCzB+HzxyBpx00J6CXlIbzhMVgdv8MwVIGIbxDilgzS7cLrygleAHJEfY2UXlEaAwQiIZugrIsS9ClWVplwht8urO8DBIU/pXsGXKLVRgctFsQVxqwnMBEIShGx1P47NPQeOZFysmXOJAk3OLgDThayde5FyjjGCfhO4SNxrxro+PNJBxzKSlw3Dn2O1/nx0xbTmbfKvsMYV8dCS9HCXlaya5bxHsnKASojN8QVoplfTl4Alyete1AVkdylf6LF+GRw46bHXzr6xNM/Z+P0o/LrksoOsdK8kccwLHiHGnyJWoupewcY8CkC3s6SeGuZWnP4fuRBKE64bXbdXy/HM76xDGCUi/CiOzcmxO4YgAeXPDt+CRh8/BRwisy7wVuqcOA80rCEGGBPQx+I9JlWG1lm/USFjsrSIp0FNEgRdkqF5WszCJWegjKO7QTCiKNRBRpnM4QyAZ77/HsexJP4i1m+Rik3TOpgR/fL2TXi1yoZfcqn3ahfQXeTXyCQNxC8AwRoSwXaZEYakeF3c1k6fUTNocmy2sfYR5azr57Ca94LxP6TGKucVn5JGBo3lAsu4gqjRGRhy2CB5XmhkECyVIh9E2RzSLXBoow7XxIZT0rZ43gSbSYtUua1dkKfowt4vudSEJ5IwgDBOVDHozd+LH/I4JFu2HyKFLMCmoih1vmuOL7k+y5U7TH8GtsjBBCCkWO9RHmYilZVXNsO7FZwXerBeWIcwNH7/gY80cTVlPfr+fwRP99LuVR5zjk2knajULjx3oTrGdBljIjbxjN9Rqzeq6IbcksyQIUF/y4LSWw7EJ4lxT4U+XZei0pL+t5VmwFpEzlFSlbT8t4r6wHgOIqDDwKOx49zkDyNUaTr/MKEq4BrhqEkXHvlT51BJ6f9wSJ1iB539adY8WlLDo/D2nNc/h6kBeQyCBLiliSTm2q/MoLUsS+DGrQvnmvJZfIvufxhNcPgW9Owcj3WvT+9YOUeIhCkpzw6m8RUk2XsuLcWlns3igQPNE22q4qh7yDlXerzEvenyPsBaRnylijuV9p2vQle9rQbavkb5Wiaetlo88o5D5nal3YKLF0MrLUYjPk1kaf/VJhx8R63jgQ5C4RKtKRrPeayMfN1v/JyBfJMHlvovV+27mqE15q39DzNkIqbQU2k//zRT6KZJDBV59t4I46mi2/x8tMKyyKlvAROWLdJ6WkalFQDGdCIF50XOfkAmoXvgJhnx5N/MpDC6+QOiApwGLq3z6wAtSdX5DnCRsWWqu9II8JeRFoYpFXkKxfWtxbBEu7FBfFJkNY/KTgSonszfKiMmjfDGttSPHKkRQvuTgqPS209SKspFBvBcJsMru36GDAORxubRE/RGttw9sWgdSRC/MqXlFvERZpKZia8Cyzbd0xZQVT2y/R7kGga6QcK3Smm0DE1cpZCE0j1E0X7ValBbxnxDF8qJ9rtdYUyLJpGynZy7TWFFJ5/lgSTx4wIoRkcRJZZK1xLTyZIAVd3lqrhH5cziqm2lsipblGVkhxllJ9Eb6vOQc9LtSd6qkIHFryXkN2Hxx5b3QTLBlW6GoQ+rAWRR0v0W5VhCD0S6iUx5AWObn8y8tHpIvCsZrmWhG8Xfh+bvuDlBjlSX1YbaZ2kfVa9SliwPY7tQXmuMqm+UgEkYgUEXciVZWm2lHeTRJE5d0npcourNqwUONKebXu0/LGkqAqQVBeSN3ZMzXvqS7UbiqHPA21x5BL/djW3JQQ4ub7CCGl6g/Ki7yT5C2leUT1qOeq3tTXqvi2tH2zK3tOndAPNd8pz02TviXJRObUCHtAaW5cNc/UPks6n9IuRFn39e3CZq1t54vlK+Kl4c7FlKWm9+As4fvyIfxaOsfphXFoboL29VZzjP7rnA2j1D1WSXbmeN6irXFbzGSmldw9J4Pmtbzl3oYF58tlldOXOkZUH0pT85nkoZ7sIzJd5W64LAzKOao4jpO9qXIZema8TDWzAkdankQ6SthfTTKo3ZNG86dtIxFklgyy0PW2fbUWKq+Sy5xJSzJVfl2E0O51vHxUcI5CdoUN4c17oslDZz3vGqvInqzNrCeHyq3jkq2sDJ2Y80LBXGtJJFvG9fJ0NufcTqRLcpLPRtIS8uWyY04eeSdrN5uO7aO2/jRmTwU903qzbDWU33xfsOfzdZsfF2eSILH118lzyF6Xv+dk508XlmzMz4nbic3M6eeLfBTJIIOPftsx7zKCJg1v/spHIGv/mDlzTIqO9oGoEOKhtbGXrFUKwynjPSgUMiDF6lmCRb+KJwS0Wdcz2bMLZTje8uFjzex8HyHmWGEkQwQlzeEt0qt467RcguUVUCOEMSiyvIG3ms8SiIVpgtJWJyjPPdn3FOE13Do2h/eWkAVI1p/ns3yPZfmVB4Mm8zqwUIX5JVhqBGXwObzgMkh485Wuv58QqvUCfv8PsnpfwiunP8R7MD2b5UNuz/KAsCy4FEspfTVTbzNZ/chLRVZ9CGFBPXjPBpFq493QXYDZGZ/mFN6z5jABU1kd6FlFwqvEU9pfoV0keCGB7y/dhP42SNgDaEdWb9ojZp4QyidLXgPvQbFK6LtL2bVapLqBZgJdI13UWaBGIFRUpgXC20BohVd6i4ySZ9n9k7DDBaEyIRBYg3iFQ1ZIedQs4UOrjhBIPgieIfov8ucFgidUk+BFogVfeR8kEBgVfLvJ6izBbWfWLoOEvneYduJGRJ08S/ppt1RrvE5m18srzgoqEmYlYELYYNtahuQVNJ+dqxEIPHnf9GR5EKGo8emye0WK9pky9GfpzGbnlKbqTGSVwqMWCURPd1bGAXNcnkoK59I4Uz4X8OPJAWnrxJDWw1ndX0p465rGgyzI/XilZszUk7yJ1G9E5GuuK2bl3pHdm+D7yjDhbRVDWf6WTdnlhVYhvN1MBNRKluY4nsA/nt03S1ASRMySpWPD/BQaZwnYiIizif/v9IkKgVW0T0folUyhuSRvNMsrYpojrYJiCdh83iDIVhrXwNrbFjeqFIq8sGSQfueJKKuEytDyUkN7JD9qbVG+NY8M4deiFULIm+pE86AIlEkgWfKbaM8DSy4QJKrLhnmOjAwiz1RGhbdqqwAbxi2iPE/UqW4kH3eZZ1lP027CfLjEiV4JgtZtpW/DZpSm1iyl0Sm0z7aXM2msB7WHDXeGIEv34OU9eRtbBVaQDCZv5bzhVunlPS2229PA1oWtIzvmLTmmj8q9Xl3aMqt/5kkHW796Zqfy5/OlY3qGzYcdQ+uRtpbkdLl0O5Ewp8Kp+pOuseRiJ5LDkiHrkYVnMoxQ9aC0T+XxZfMHJ9ZbJ1J0PXSqZ9vH1hszEaeHSAYZzLfCXiF5V1XbITst7nZAWu+SfBqygohcWSJ4/6RAVwV66mFQlwivQBbk/TDbbPeosbHF2htF6eTdd1UOlTUl7HlkB7K1asiSXs8d1yLeNOe14MoqDkHQkBBmvTqsq7aEQilt1W4oL0FfESqtsL+GLEtSrpICpM5/JEAonEgCCHjlV0q0PB5E4ljLP7QvIvLksC6YWuytMCLvDT1XgqfuK5ULFIsJRVprxIjCtFQHeq5CSpR+2aSTtxJZt2jrzaYJVPvGWMuZ7ZfQ3qbqZ2p3Wyeysq2QrnkBSTFXvReAvj4YX/HkFybvawtXAr0VWFoNSrUEQuuqbYXZAsEbR94VVhHQpJa3CIlcyAsJEjbU7rrWWqxtfasfK4zK9n2lp/8iGCGQT6oj3a+82nAmPVckg9zprRU2JXj6kf3vIRBOqsuSuUYeaBoLuk8WM1v3dp8inVcZhSLtb5izgpf6fkL7QqP/mquUhhSPAoEkyVu98nsAqe5EvFvFQvkWIZcPdVS7SdFJCHOo6iWveEEgJPPeRuqvjhDqpzTkCaR+Z+vVto9VfuV1lDdGbDU2q2xst3IScXawfOpLNgWrgFu5CU6UubQuWWWgZD4a11ZBsh4mWuOsbCMSaj1Lu5XnOilpGtuWOLHlysuCp+s5JdLFhj9pnhCh0kMwAgwQPHsq2TkrJ62VwYUySAaz5bDPz5MeNnxJaVsySHOe9Zy09ap75Rm/RGgnzXlKT+17KsLRylyWOMjf04n4sXWs8ndSqtUn5OmrfFoySG0iD2JHWKe1hoGX+RSmbeVuC7uWnkmPj40gT4qtp8wnuf/5NDbyDCun2bQKuc96z8rnx+Yr/7EyILQ/t9PYz5NGp4ON1IOVxdd7Vp4MsvncDi+ZvDzW6Xx+7s6f2wgxdjLY+fWltsvpYjMyz/kiH0UyyECeHbLq2/h3q5haRUOQwED2bTc0FXmihe4Y4fXQTTw5sT+7d2QQWhN+/4pF/AJ/Bd5irj1p5oCeBH604u+VwlgjbGw7SIjrV5y3vAqkvEkxl/X5aeASky8IYT9NPCnRTbA0aR8ieTGt4AWRBUJYjjxcSoS3iyyYfELwOtGCJ8VNlvEdozA/DburfiPIWpZmHW8Rq2XXFivQbEG54ePexwivZa7iLf2H8W9je4DMXTrLi+pEgooGsDwrCnhi5ii+/Vr4RXwM35Z2j5IaQcEsZHVxMKuLIlDu7aJaSuidWOSpLJ2LgIcJgpQ8yG4p+NfTy4NkEN8XRCioX0rYFLmoviLBuYr3TlCIjTxzrHeHCAwIBJS8VzT5imhcyfJxrNVgIquXIcIr4Rey+tq7D656AcbK/llThDdHASwX4IoReOrFsK+TvMzsHjXWAl3Fe4jMEl63PkUQHLsJniyWwOgmeKZImC0RQhprBO+hniyPEo413kXczhI8hkQM9Zq6ljeXPNmm8N5JF2f1VsV7Naley4TNya21eZJAZswQ9veRl9uxrB41R+3O/muPrgYhdHBn9pHnHrSHu/Zl1+0i7COm+UNpiuQTurK6kBKnUAWyck1n12svIgcUEx8WqOfJ+0jl6MV76AzSLliUs7zYPS3UdysEbzR5/Uh5HSB4+sibLzX3q2zdBAF9grCPmdpCIQ39BK9KCG8nk3deivdmq2T51f5GQ1k+1D/mCWNMZL8l/kRg95j62Q5EMijiTMCS650gY471eLGh90rDKjwKd1XYtcaxNSxJzuom7DNm04PgeWfJhzzhIGLbWrotrCHLktP5fMu7RXPTZqBy9BC8gUWeyAtlEL8OXkxYv7V+iijXt13/5elpy2GV4ZJJz+5bo3lJc2aVQI5I0bPpimyyZJkIrH5zj+pI53qz+xYJMlne+8uSW3lF1OYDgoyuvia5omieqTJK/s8/T0bKSu6jvq71aBAv/yhMuJyVdZFACg7g19o+/DopMkywRBi0G023A53GbZ5QzROHeVLiZOM/n64lfEUa2NB6tZVV/tczZubzYz361iPWbF7tvcpTnizaKKGxkevyBkebX8wxhR+WaZePRUqKuDqT6ESU63eexMz3iTR33JK1G3lu/pk6bufmSAadGUQyqANsB+7U0bQgSHHTZK3Ftomf6JPsf7kA4wkcbgVSyFqTJAQV8ISGJr86QTmFMImlQJJCqxXCsCQk2PxI+dUmuhDIGlmQ5JVSyf7LRbefkA9dJ8t2jWAV6SvCShn2r7QvVHkrk0JxtGDKMndRwb/Fa6jglaWG828Sci3oL8CxFIYq/thwN1yVQl8Kyyk0m7CzABUHw4nfCyBxPr9S3OzkLW+pLnNOlhtZr1SHo9m36qEXT/ysJr6CkgSGyzCQwFQZlopF0qRIqVCiN22wiqNIQtOlrBbKDE41eXG56b1TnGMlDe1Uw29q2XRhIdQiqAnfClJaFOzEq/2I7ASq/qn+KiGoP0ujnyBcixwpZv1lMLt/AE9I1IDBBJrlTLguwkgBSqspu7O0JAj1Ehan7u4iA6UW5YqvOxEddtEqtEI5rOAmj4n84m8FdzsOIVgTVbd2IZMwoba2BFcnQVHeOsqr6sgqOXKht/0Mk4bKYq2UOi4PHZXV7juEudaGZNm6sESeJTKVjr7t87roLEDkXfxtfVlrTn7xt14rEqBsv0zNNbq3XIK+AlRWw35F1jtJbSehWyGyEgAWCG1i92KS4mWtuxYaT9bSp+eKPJWV13rvJEBvJcE5SBruBAupJfAS8yz1D9WjrlNfsJ4CVrhRG1Rpb7/twkYFeHt9xIUHOwYKueNao/KKgsaOQpi19midWzLXduqHepb1DrLfGs+STxTGVKA9n/J61bWV3P3QWeGw+cp71dg1zc4RIqu0nm10vGhdsASYvCklR2mekCdKkUAua6sAze0KX9JecA4/typUS3Vh5Q3Vl+ZxzXHy6uklEG5qb60VnepHdSPvmb7s+Gz2X2G4NkRHbZ2fBwvmWrtO6hnKt9rd9hHlU3Oz8qM5t9NaZ9ORAULfeTLIGhYKBMOgDHgp4S2W3QSvYQtLgiqvW4H1lO6TXZ+Xd5LcR+mdLM08+WK/88/rlP7J8pVPy96bJ5CE9Yit9fJ1pnGqurD9wRJc+f9bmVfVn+YTeZi3ctd1aveT9YfNkGv5tjkb2IyMdL7IR5EMykECvCAlW5BVYwd+wZXV5jBwGf7V73PA1cD38XtGVKvwhgr85mywcshKnhBeBVgBkr6g8E7h9wSS90FCeA17axmqzluDjuEXlhkCgbGC9yiaBK7M8nQEuAXvhTSEJzcOESwwu7L/w/i3XA0TiKOhLA8H8QTBcPa81/RD/w7458/AlAuLpULFpOT2ZuURUvzeHx/vhUfmodLlLSldDdjZC8fmoL8Xnl2Cm/tguRteuQduWfYhRY8vwLePw9/ogifrcEUVDix5Quim7FlzBIGpSNgPRlbFalYHBwnWepElb8YLOZdkbdGbQMHBYAX2XQGVKvSPQKkLWjsTXK0XV+4jqY1QWD6OS5tQKENjibRnhG/+yQw/fuQ4ZWBpcYXnk0AyXQ6USrDcCJ4V1Swfj6TBS0F9Re3xgjk+ir9OhI5ChCR49WWfF4Cbs351HSHkRX1sV9Y3L876hLxlrgV+ogy7x33j9Q1BswGLL7b4Z0AhgfuAWee9U6pkb0kbqrF7YpbmAByegIFmeIVvCai2YGkyhBNKuRdZdYwgoJLlVZ5hEtbURwsEb5IegmIh4U9eW9qfRkKcrHJaTBezdAoEr49Sdu8IQbDrImyADu0EgzxJZM2sE9pW++mIDBKBtkqw+GpxlOVUXora+FlCdhHfF2YIwvl0lm8dkxV3jrCPVRk/h5WyMgxm50USpVkeh/BeOiJMNDepPfoJ40aCu8gxCenak0oK2HAfvLoP7j0IvanvJxLAZQHvJby18MnsGb1ZXT1LIDIHCZ5cU1kehwh789hNqcv4dlcoXBchnGuaQIKrrdIsvS7gyl1VllcdR46srhFOljCUN6gUzN34NUEej+UsT/JCaxD2zrIkqvZXGsTPTS+w/WFikQyKOBWsAiLvCM1/FcL47SMYLkRCd+HH2WB2bgm/Hj2HH0NL5npHu+CvZ8polXLi5u4iK+Qp3U8gKzTHNwn7e5XNeZEYVuGw5L81cumT3xMH2g13y7R7ttj1QuWyBLqOq4y1rK4GCCFF84TQ1qHss4P2PStFoGuNl+Gvmj1nnkBs1AgGSn10j8qiOVxtWMN7mu7Cyx9KW3WsNrRlJtdOZYLcOkfwZLUknvpXnnRTHmrZf+1fackTkW+qzxLta7LWgpqpX5GSS5wY2qe+Z8PA8mRQF14+3oHv/9oPTmuR6qMLP8fLA15rh2CJT9ufzjS0Tue98mSUsQSpNfjYPiu5X2npvjxJkIc1kKiclviy4zBP7ggu950nqe1cZckgm5bGri2PyA6113ok0plAvj7zpJclPm1Ipp1TNcfI8J7HZgiXTvfatpfzQA1fPzO0b03R6TnqD4UO5zdKSFqCvEC7IXY7EcmglznyndROJGJAJejswO+DMliEbgeFKlxUKHFpTw+zJOwrlHihucxul0IpZVe5QHF25QTrkZSBtUmqGIghCMqhOt+aUuyy0I7Ee8UMVWG5p8rqQoOVZkqplnBxschis8mlid9fqNSAS8rgWt6j56IClFvQU/BvI9uTQCub9XbjPT8WWlAp+MU+BVwKw0WvFE85uHSgRN/OPrqemW57W5gW7bxFTpBl5NIemFqCctVvIlwrejKouw59Pf51430V6CnCQDcMVhPm5x3TLRhNYEcZDjRhrL/K0aVV+oZ76G06xhaW6anC6iprr8KVhU2LXDXxnj2HU79oL2b56ivCvoEytbTJ3q4ByqVuehJHmpTZ2QWX7ytQqUJt1Lc7YwWo9UG5H2rDsFSDtAlFTwbRM8oTA0+t7VVUbzrqWSbWLHdJu+Arq9gM7Raq1NxTImzYrIVCe8Ko7kUeVYCLusAlsMvB1Crs7QZX92E785nkuCeBuRR2l6BV8O0xvAgXdSdcMtTNnl1VUlegb7jA9OQ0yXKZS3f30mgt0zu1xFDDMQwMdRdYTlP6evvo7llhJllt2xPHemfggjBnvXd0vkz74lbMldHWm91/hQ7n1R+tN5Dq0F6vfiyBWkrOICEkTePRCjwSbFQOTFrKmwR1jWsJtqXcsXx6EmL121oqrVVX+dE1EtCs0GTHZt4LRXmT4K88aJ5az+JmhUZof57yVMIThz1l3xfLuev0rbaRJdfun2P7kRQ7mweRJ6qfFu35U/6LtAs4VpCynkEVoKeri1YWwGXTUZ41PtXGUlStNVpKoz46b/Nu86aww1MJ1Gcam1U4tspiHXHuws49Upo0z3QRDEZDtIcjFQpe3hgqem/gIjDbgLlm+9sVoV2xsMJ3JyXPenwmtK+JUt61DmqelNeniAeR4FYh1BypOUyEiK7LrwG63pIRWofznlJ2Lu2kHFniS544MoRYmVH1r3Io5EmGsE5kkOY2EfWYctmQMUusWA9MkVRDwM4S7Cx7uala8PJEIUlwSdJmuHBZaVPnay5xUEihVIf+1RC+nfd4yK/d+fqRErPetTqntle57Ppo+7Gtg7wCLXJNYXPqV3kySEbinuw+lU2kiwg7fSyJkkcn+flMwY4f9YmNKK923Sf3W9iowpxf//PPsd/569ZL/2SEjfKaJ486EXGWiN5q2HpYL/+WlMnn0c496+F0CCFbT/poXhGxbOfJU6W1WWNTHrb8Nn/bic3IPOeLfBTJIAO7V4zeZNTEWz/24jvgGJ4Y2VmEQh9cNwKNJvRcDWltjCt+6qdpFSv01MZ5+8G7qKzOsnRwmeeWeun/wr1rmyqXCF4qx7JnLQJLJf+GrV3Zc2cIHX/F5E97iFxVgZu74IZrS9R+6lU89qVnmTk+zy0/U6E2tJPFF4/QU3EsHW4ycwgu2QWz89A9AN01WJmDUi80l/zm1Ud+DM1VT3QNDMDUBNQGoJKRFdfOwMAwdCX+1eg7rx9j7oafZcddv0u11eKFLI9zWZ5lyXKEfWlcVpZdCey5GpYehq5x6ElhoAa1Qagdg+5h6Dnu+ZXhBEaGYOiyXvp/tEB1BprHYM8gHGvCTT95NXN/+gg/9f++lb7Dywx/+fuMXwd/+jA8PhsEpHHCnjeXdcHryjA7BzcA92Rtc9sg/MLtF1NeOkb1tT9HcsmbKDQWYfhyygl0DQ2QFCDRCl4GikV/oFiCtOHZuiTx7FmhzBV3/Uv+xvcO8CBwdBF290FahkLDW+h2NIJAojCrKfy5XVm9HaBd8b0IeCq7fiUr296sb8g75xvAHmC0BL9wLZR7YHYBvvEkvOdmuOtHcFkZfjgJ40NwcQXm52HnxdA7UGBgsMCxbzT5qTdWufJv3Ep1z3VQ6qbYU6P8h7/D6C2XcNUb30Z64Ps8/5vfZezFBUYrcOONvTTm5hm7+vUspk/w+EOPsdT03hFtC3wBevtheNp7rhWzcshDY5Tg0WQXwsMEIUZ7P2hsTBIUAlntRLZo7Mm7Q6GUqwTPkCm8x4gsjiJvhvD95IEsb3O0KwEtwp5a8vawhFwffl+gYdr3l6ln7baHsNG40pQHSZqlMUW7dVakYA9eATtkni1hdZR2z6Q0y4cIlkuyMu7P7pkgvE54KKuHOcJeThKkoT2MTi7+CsOU140sa7JwL6+E54zTbn2VV8AYYY8ged0ME0JeFTO/TPBYgjDnyHtJ+xz14L1/pDzIO0qeSCIYE/y+V33ZMXlydg9dztziIglPrlmMu7I05dWTEDwmdwCP4UnzIr6/yrtT85/WAHlSimRUPe7Knv8I24tIBkWcDFryNH8qVEhhL4P48Xsxfnz34ZfFpOCNPn39MDwK/f3eZnL0ENQO+DnpCL7Pazynud/WA8dayTUP2vlY838vfl4fwM+F/YT5WiSBQuDlpS2yRYpgg+C5Ke+OvPKo9dt6xjTw84HmMnkFWmOI5gRrANDapjoewa/pQ9m1U/i1dJnw1iqRD/2EMWlDOeQB0539nyV4BeklDC3C9gQyqiTmuDyR+/Hr1dUJXDkOu/dC7wgk/ZB0AdUuKJY9AeQcDgdFrzq65jI0V2C1RbICk0/B/ENwIPV1pTXXhoB3In5saJeIOrXZCaHJtO+5pD3Z5E1kQ+0w6chApbaW4aQn91Gbiwwawq8BIt/y3kjy6tL6Y42UgtrQGsfOJCQ79RLWxPwep50Ud9tnOxEFVknfKBmk/r/e/TpmjVL2/jxhY4mRPGnQKV17PN/frOHspRIZJ4PkwU7PsG1hyRBLCIlI7dRP8sTdZtZsO6YsqdxP0O/ORP/Ml7lTHi3BL/I1kkFnBpEMMpBLYEII51jBCzM78Z1uHC/o7EygUoKren14z8iOEgv9PVx21UVQqkL/xVxUfRqWHAutEoePVk/Yx8R2ak1mLgmbGkvZENLc72Wguwh7KgmvGq0wdOUu6n0HmFxIuH5vL4XRQVqFaZJSk9VGi4V5x/AwrBSgOgzFPu/9UeyDZhWSMlSOZkJAAv094MpQq3qLHngCYyDbM6erAN29JVZGR6gkYaNGKbGanGR1sBOGyuwqnkeplCBJIalCq+K/C1UfjuUKfk+dYgKuq0CzCNUiDBSye4Fafxe9RRi9dISexjQXV+DykQK7qymTWbt2F6A7DQtKbwKDBZ/vEbxQlOItXXsvqlCcL8AVY3DVFbA6Bzte6SXanh2+AtrEONdeOpd6QgggKdA1NMhQVierLagUCywWHc2GYxW/V5KddCVYOMLr7gUthpqIJeDIWreCF3ql5Hbj6+ryGgz0wbMOhguwpx8eKsNoJbPWVqC/CsUV6O/21w4OJgwWYaQ/oW9XD8muYZJKL67ST7lSpjjaR/dVu3FJP92VgheEEtjdV6DUAkq9NHq7WG34PZEkTGK+S0XvpbXqgpVOxKG8naxFD4LgJ4tl3fyXNVYCmtJUbH7e8msX4aJ5hv7Ls6NA2JdB13WyXGHSS2gXaK27ccFc05N4D0MrEOq8hHL9lhIjxUM9UV5O+l001+mYlBgr5HYTerDIK40Recno+Xnrj12MZQW2dZO3nJeBessPDSkcNl2VUfUlpcmGjOQFw2YuHauMiTQqEjbKt1Y0KRU2/9rgXM8FoFghLTTWnqs2Vf5sXysQFACRkcqf9WazApqUrbJJUwpKJ5fvrcRmBd6zIYxFnD1YJcRaiC0xNEB43Xl/AuXEewRWit7Td6QG/YPQrEN63F8v8jwP1+FjFb28J4hV9KyiJNJ2gKDkFQkeQyJpFWIN7aSO1mTdZ70zRWDkw6QahL3aNF/n69L+zhNMlvSSt4kjkOAi9K03jSVNJGOKQO8iyGmrBNkiIawteWlG+bBeslpP+4D+MgzWoHcYGIKkB+hOvHAGQQ4qFTMGJoFGAivglqB1KLwJLe/lk1fobX7sXN5JiVYb2Dqx4TV2nbLpYepU5bVkkPUkskSk8iFDTMWcU7kkU0jPyLddHnZt73T+pUJjQ+SZ5KqNEB/5drFki+07m11LOo0Dm9Z6REleBsuTSnkyqBMRZO9dT7Y7m+hUdov18pov72bbxY6P/NjY7vpRXtYjzrYDm3nu+SIfRTLIYC/eq6AXv0/GVXhLxT78/jvH8QLOMLCzyxs/ajuh0iyR7LoKCqMwsBdw3gWjawiWp+naO8jU/+9xZgneB8cIoQQQrOg7qyVeTZPn8ULLIr7zz5p8lvHkzXMt6K7CJeM1el/3arjl/Qx+8ccs12dJXvkzMLibQuVemHueSv0F+pa9WlHIvFrq09Cch1LTO7PMzcCRg9BysLgMA0fh0AIUJjwRUweW6jB03Fv1u1LYtTjBxV1fIXUpLTypQpZveVtA2HvDKnePOfiTH8KBWRhdhYLzGxTXSrBch90VOFyHt/T6cLWJ/fDkgSW+cxR2NWFyFYpHfMjT/geeYqQbunp3MnXgfmpDUBwd5ZKRCWoz8HgTbh2B48f83gRF4BUNKKTBbXpH1u7ddXBTh2BuFWYPwtSzsDgBy9NQ6YXdN0GxApWat3AlBU/+lLp8RTZX/LX1eSiUoVLj2MX7mCQL+yvADRfv5E8PzbN/ZZF5vDeCSCBNtANZnxwivDWqB1/3JbwFFgLJIM+JWcJ+TatZ2YoOVhdgtQjzxyBdhckX4Nnj0NOCw6tQrsNMyZMys8/C0IspIz2OuXk4dN8qw+k99O99mqRaJK2XOP7EUbpLO+ne/y147iFazVV6Eqi1YPXYIq0GHP3OXQwNLdNbT2ikbs1rZU0Qc14gHK9BMh88S3ZkY3GOIEhPZ/cNEmLxjxAIjL7s+gGCsCyCLDXtK+KsRPDuKeCttgXCXghTeO8r9eWDwP3ZMy/Gewk9b+5z+DlC5NwsQUlIs/xrEdmBFwjnszHT3wOVxSCcy8XfWni1gMsyI4+dJuENfc2sD2jvGZVR+xZoHqkSNmIuZPUiT6gh/Bjpxfe7OmEcq570f4mwj5mE1VbWBoXseUWCh9MgcLAOh+dhxfm+LNJG3kPaY0xeBi9k5ZojbG4uC32SlUP3a07tJYRBaOxA2JBf3lLK0yLB81J7Skn5S4Dj0z9iYbmxRs5KGCnSLpioLrSHivaVcvjxCb6f6tpFwv4l1utB5wuEvY22C5EMijgZLHEugkEeDlbZlodJw0FhnVhHR+AK9L8TWubbhmjlCWGF0WrOsXv8iPjpJhBPZXwYfKkAzWwAprTv6yNSxz7PksCWaLAEvYgIEVDy+NVaofLmSSxo92wSgVOhnYTSM2QA0Vo/SzAiqn3kBWnDv9R+Wp+ssUGeUJqT5M0ij5oS2VtjHcxMQd8xKFS8QcMLe6tQbGQLmMtuWvJGtGYTmunaK1cbq5C6UJ/Qee+Z1PymwzH1N62bMiBpjhYBI09VfafZfSsEUkRrsA3ntiR+XjHOk5G2ryhfddpD8rRuYNLLK9YyVqidTkeJP9n1alvViz7WM8heqzTzfdh60imfeSJto8iTMtawtx4hlObusd+63vYnS+TmiQXbp6wh60yQQ53Guo53IrMxx9RfRd6qD0N7e63niaK+u14e1suvlXHs3C65Ut5k+fmLDnmx/fx05AZL7J1uGmcCkQx6mUObe/YQlO0GXniv4RfZtUW6COUCJBUoFBJalX6aaTdNV8WlDW/+bhVxjZSkq8rC1Nza691tDLcd2AWgu1hklCbPECzDdgEku84VM7fjAjS7ShQGB0kHriSpdEElodV7EUn3LtKeEVg8iisnUIXVBjSy2W5lCRorUM5mkLkZmJ4HV4CZeagvw9GGz6tCbFaBlUUfstQLOJYZfXE/zgVFP6F9A2mRFRI+tDBPAk8f96EtC9lzlgiK/gpeEbxpOVMm5+DwfJOHZ8OGw60lXzfTEzNUiwUKrszczCxdZUjLXfRWoViAZxNP4JXxQkwC1NKgxEoJrON5ndXFRZoLkCzMkswewy0chdUmSdcArv8KKHVRaBVo1RehUIK0RVJxuOYKrrFEsjgJy5NQrJJ011msdq+9WtYlsKPWS6O8vLb5sRQ/CBN2Ga+0lnPH1FcUr0vWnyQsWmHHCsNLGeGzuAyNlg+Pm1yGyXr2uvSm70+NEhSakC6Dm3Ms1mF6ImVu/1HKzaMUqj6scG4S3NwyK5MHYPI4jVbLLzQOlpeaFIC5wxMM9BQpuYQUt7YoS1BzDpImdFXbLXTy0pMApTFiyRFt8ihhNiGEMsm7RX1XhIaOKZTJbnpnY/n78CSCPK2knE8QQpd2ETw/tMCKdHEEby6Nd7vXTW/2fy7LV7kclAqlZQWuvJVUew1YZSCvCMnjZdWc03xTpH2REqHSQuPa52We0LfUNhKqIWw6Wjd1KnLHxpNj8rCcwmI9I52zc+q3Zdqt3eonynszS3eeoKDlFb6WuU91oLJKGeoy9WIFHLVtw6QBsLQ8S301XSuPJYqsUqb+JEFYaUF4k5vdrFZeUBLqUpOejm33In0ygXK96yMuLOTnJevZIDLFKsGF1MsVTZd9WubjggJ2qmdaIsh+0tzHKvI2betpIXmknHgDWdrqrCitl6YlL5S27rWKo4giyUJ5ZXK98WMV4bxX9XqEGPj1apn2+S8lzO8tc2y9cloZQnORCJMCazwOK3i5YnEJeha9U3ziIGm1cEmrXbNVBTQhyRgntwqNRruHad4ryOatU9uLjNAaiLlf57UuaK61bYm5t2H+27aG9uepSBadSAStzZItrBdW0aS3noeF+sDJ9hR6Keg0dk42Fm3Z82PFto/yfSa8Rqzcsx7yY8i2U74stt+7U/zeKu+T9YiM9foWnDi/nWxMdEJ+XjoVmZLQue41H4gwlZd5/r5TpXmysm4E6xGD24HNyEjni3wUySCDw4Q3P7TwJMULBAv3FP4NYRcDr1zyHjXPPgRF16J4+FmWXZlDD80x51IoFFlZPMbs6hzj+4r8/qElrwjhLfgQlFl5AxSAUqWP7uoqx1Z9J9qB38tDC4cIi6QKPzEIx16Ex55bZN8fP8y+H/0S/+O+/bQWW9T/818wXO7mxcWjLKzOMr3cJF2EyRa4JgxXYX8dVprQW4Q9Dh5fhudXff6mgWLTK10DWb6XCfuRSHkcWoSr71qg6fw9El6kmEl4kFfCAD7NPrwXxbcIe5NYBb6J3yujDvxU9yCt4RkOzsDTS/7YQ3hFvS/Lx18fh9e1Up7/71/n956a59pleOV3Jjl4GKYb8KiDVy7DM8Cr8Yr+UykcSr33Vxf+3DzwtSWY/0vHSgPGnr6fHbVDTDeXaRSrjBXLzNa+SjMp8opihe+nTZIkoekclxRKHHYtFtImXc0Vmq06paTAxcUKgxzggSz91RSqtQF+cnCeQ4e9V5C8A6wHRRHvoXKA9jc9SXnV3gpi5g/iy9ICHicQEk8CR5uwfBDKR+DFRTiawg+PwLNNv7fJNP7tdPUUmg0Ybfk3u7mCb6e7l6D3Oeg/As2i70M7ZmHv/QeZOjBBYWaeJ2ZajDlYSqHygh9Pf/+nr+PFh+6huWuU8twUvSvNNQ8O9aFiEX605PvAPP7NMqt4srBBqB8IgpS8L0ReyLonAdZaaacIZIb68qr5SFgsZHnaRfDMkqVQe4jJa0Tk5lB2n/q+hE21lWKrZ7LnW6JZe94UgdlqIAgc4S1iJQJhkuD7/HKW/mL2W3tBTBPmEQme6hsiHJW3HsLb1hayetfz+025RYwtZr97CEQsWRoKndP+FiKI7H4eVlF81RDM1L01WPNhShAeu/B9bgzveaU9f0TOyPMJwma1GgtlfL+pZG3Wm7XRFIFw0ibrCruYJngZDWbP1rWqn4PHGrRaYQ4UEaQwRtW1PNTk5XSMQGZKMVPYGtlxKZbdtG80O4D32NJ92wXlZ6M4X4SdiDMDEecLBAJ1lmCMqODH9EG8MWOA7A2eLehegt4mDCxAbxXSFKbm4Qn8WNGeYZ0gxdqSFSIo9LEWapFRSwTFYQW/nmhfsQpQTj2B0XC+TEu07wnUMuksmmdo/GvtsGS+xo/mkUnCXkN55Q3WV2rSLM8yRM4R9hdT/VcJe7ylWT0eJswz2pOmL0unlqU9l6Ujr8hVk84S7XtUyltIZIaeXQeO12HkMAzOQk8FSiUoFNuVVsDvs4iv66KDYuo9xo7PwRPOG1q0nohM0LOtF4/qRfktZ3nVeiijmfXk0TxtSXgRldbbSgYbrWGWpFK66oPq+1orRShJPq4TjMhHCQZOyW+SDbR+5skTkUAKP7R7220EqsP1FFf1Lcnq+Tre6DOUlr4lT2n9X8cpsO3+PKGs4wqxk3FQhr2TwRI5Wl/tf523dZMnwKwxRsfzZFyeTD0VrBeZ7rfkLJxYH3lYolH9UMbsjeTHEkIbza9tY40LkZsKa7fzpe6zedZxzZfO3GPLutG+Z/vLqUitrcBmZKTzRT6KZJDBLEHQUZjHJF5BmCUoqpNA0vBesAuL0EVK6cVjfnF89EWOZeks4ENKrliCu2dD2hA8BbQnkSa4QrlKpQTzq36xkGJhLd4rQFKCywfh6H54cbXO8z98kZGjX+LuA34j5ou++ejaBsPH8cJBihfQCvi9jx7M0urHvzr8brzyNYYXElrOD9hdBIUpj54GHH3e516KqZQ2eXNI6FrGC4fyWmjgCRgJObLKy6ol74K5SjeVnhmmJ+BYZkV6gbDJbAI8twhXA5M/eIIfrGQhJouLTDW8kPFi4j1gJvCvni8CEw4exS/YR7K6aQGPNWD+Cd/elz97kH0c5MWsri4jhM6sAF8hWMpem9X3pGnTMnAN8IarPKlTwb+5rVTt4oquEiNZWRQmIiFD3iV9BEufBE4IwhCE/jqVXdeHJx/k5n0U7849Oe3vm82OT8/7+w9jBHzn230oDZ4hPUC9AUvHg0cLWT3O1Wd45FkouWAtOORgedYL+G8Zv4iJ/1Nn554axdIsFZprBKGEp2LWNiIMp7M2VYik9aiRIGUFGJEUS7lzWniXCIuNPEK0mFm3eV0r7zYJX6pj/Ve9QnC9nyIoARqnOt9NCCWTwiQBsAokCSyXwrMcYeNj6xWWEAR/1YvqvAc/zkVSqC818P3HekuJmNBCL0VAebblVkjavMlviSAo1fHKnqN97wFZovNWXgfs6oHl5dDntC+OrlEIgsvqtUU7oSRiD0IYxJzJ7yxh7lEbvEh4Xby8euRdtWjyITJLoX6q5+mFdM3LxxHaWBud1s25gew5Ugow5SoQCHJBc6UILNW96uVseAZt5fUR5z9SAhlgw3PIviUvzeLXox68saG74WWGvsWwxs3h11+R7Kd6bicPFn00L9tzIsjJji2SkVNk85nzBIWUe+3FI6WxRSCblmlXYqzCVDC/7fwEYY7JK2wu992pvE3Ciw80V08RFHnNPSrrPEFekwex1j3MfTJwKgzakkHLBIu/FGo9wxo8injj1vE56JvzbSpvWauwao1V3WjeLGV5PUwgurpor0Mp7FbhdJzY/pKnoX1PQQjrtvUUlaep5AQbli3SLu/xYBV31X3R/BYZpHpP8e0xn9WziEkIRpu8QmxhScbyOtesh41401j5xxJ3J7tnveOWdFJ9nipUzOZxPa8R6x230fLnSVb144I5lidj8/fnvW6UH5v3zax9eS8/9TGbXqdPfr7o5B3UqUyd0Kkc60H1ZckgjUNoN4ql5h7Ni7pGKHT4pLnfG4GecaY8z04Hm2n380U+imSQgciOyex7Aj9Za18Sq3Q/kh17jkCA1AkhTmT/l4DnXvQKtgbyDMFzQCSPFshj5Rr1ethjp0ZYbKxitbKckPTtZJWjPITf3ubbE7A/9Xn4Hl7gmcQribKKHcvy1sAvwmX8QjxAsKjP0W6BKOd+20WjhRfirMVDCqYIHymlJcJiWMV7Pc1m5ZTFvZtgidMgqqYNFo/Aat2nc5igpE1kv4/jvWFWG/7/d4HFVvDumnHw0IovxwTBWifrjCYXkTEvZt8VgtVpMas/KXkr2XN7svufI+w1MpSlv4QnmZ6YCILKKNC3dIjHZxYYyOpengpWwG4RBKRpAulA9tzx7Dlk9SjvCFkulwhK7TxBuFMZJXTJmiYCRBbGMmEPKAh7QIlArNag5mDO+WvnsudIgXdAsVLmoj7oG7iEXaVj7GeVImG/miIwtguWnoeBlq/3Eu0CpBYKhcxpUdLCrlBOCbXqN+qr5Sxfh2nfzLFFu3AgwtJ6wen5IqrUPx2+jy0RyBl5iOSFjGMmLyJCpwmkSiO7cIngSbMTeJYQatXKyqh2IXtWf/a7gp8rVFddhD2UFOKpehsiKDYVfD8VSVHGE97qk1Yhcpz4Kl15eVnlsIjvNyKQZNFLsnocGB1jemEGVlZJsnyqfyr8b4iwP5T6RE/2mTPlkYCfZOVfpN11eZng4SVlTP1hjODR5PD7Q4l803yn8SUCUGW3bvtzBKG+j3ZrPNm5wex6pVMgWMAx9advl4ArQtpsJ0O3A5EMitgMRFZoPGguWSTIFCJOuwl7DGl8aB2f4tReD50UIxsim79OytRylp8pwmbSMlJpPZB3rg17kMKsuU3yjBR4S1jY5+m3ZDyt4SvmfNrhfh3XM1WvTcL+awovliFEhiTlX56eVtYSIdZtyjSTfSTLiNTL7wOiMukZCe1vxTpOaNcu2tdXGVzyyqL1dlnGG8omCIYLefI0TXqd5hk7f1rFU3KovUdGIHtN/j6VOe/NonXQGoxs6J/1llAdS26eJ9SzPN+07rRo96KxZIfWGRkp9MxTecbk83yq+TlPeJwqTdWNJWbz5IlIjyrtG7Jb5JV661GSJ5X0ESG0UeLA1kH+PnvOEn9Fc0x9xNaNJWk2QwjJICRZSONQsHmw9eJy91vC2ZJl9tx6z8+nu57XlvXSUj4tsavjVg/Mz835fFiCUHWq56hd7TPWG+82HZX7VEaEM41IBr3MsWq+tfEnBHJImKV9Q2cIykYeCeCOtR+TkqSFREJBGZgudeMaQWmQB0FKmEQKwOpKQtIzTJ2jPAYcqEMzWyUG8YuOhJrjhNdZz9OuEA0QXuk6Q1DM+ggD2pJB1npC9nuJQCBZskHhdhIg5PWj+hqEtT1z5ggEmVx49Zxy2mJ1MuwTovLIVb2Bb4/n8HvhJPiNfl3qr92flefpeiBU5PnVRXgttCavOmFfoYL5LbJjb1ZXLxLCVaqE159XCaFBy1nd1qaDojwI9C5PMDnXWAt1m8ULQZqcNSkqtGfe5NllaY8QPJBSgsdBi+AJ0pv9F8EhomCFsAG1iAl5SshyqDAnWUoLhDeR9OD30u5ZCm04SXD1FllQLBXZ0QOVvt2MFso8l51TPygmMDIKKwegNyu8+pi1mKVZfstZfjUeVFe9Wd6soF0239P4vjBIu8BhBQSNrxWCwAeB6JkmEDIO+CGBBFa7dxI4ZvDedRoLDt+X+swxXHjdeIkQNmnL04XvDyp3HU+o2hAuhZFWsjo5SgiNU//uIQj+xaysGpslfL8eIxB2lgyqEsgXkTES+NY8vQjhoPKak5DSBfQNjDBf9sF6IojkDShFUp5LmodEsMgLSHO1vKKSrLwTBCs2WVrDhJBVCN5DEijU38YICpSUkIHs3AQhpM0Kg7L8Cj20C/dk19UI3koqp8aS6laKYhNIE3AlTwZtNCzgTMEqqRvB+SLsRGwNRGCeCiWCUivFVlgieKyeCvnxJwt1ft61RIu8fpRPeVqIrIEgt+SVERH21hJvn9XK3WOVTxFOIqbtnA6dx47mW3nhKP9SzlYJXrM12o08NsxLsolI8AqBjGjg5y15ZMsTyIbdWa8VzDOUB8lJNhxY87aeofq0Xj0ig3oIId7H8eukyi4iwa4d+bqyc5RVTPXfXq826gTbd/LGHFt2HRORk/fyUD8RmdWTXb9IIOgWCTKJ1n+lZw0MerbWki4CcblRImIzBM9G53Bbz3mySefU91UuyaR5SLazZJCiCgSd3ywZlC+P8qb6tXODzXe+XPnxnpg0CnSut/Xq0pZR99t78v3O6iS63hInnQihk3lPWSJIec8TXflnSc5bb461RKA9vt6zbT7zxKFtk/XGaqd05LG9ndiMjHS+yEeRDFoHmxHAtfjJ0m8naXtcwo9VrGTN1uLS3bebheKD7GxlITfmnBSrCtDX7ygd8TTVDoKbttxl5wn7iGgyVkiFJTC6s/MzBKs4tFtjtK+J8qC8i7CxgoMmDOWl0SE9WRJtGImEAwiTmRaAQrmbvV3TPF2AqSX/PHmXKDRoAa/EavLqJ4TukKU/nz1/2rSP4s21QOetgvL2coT4dBEmmojlZaP6aOG9UETkFEzeAPr7iwxfVOSqH9eZIFjVLNstYW0h8Rt8q47SrJ4aBKHPCmDqY0N4QU9toY+Ubrkwi0iRYl8meHoUsjzYBWHVtM/FXRVco0U3rbV9axZyaS89/TA7+hyT1RCaYxeiUjGhUOxiqLC8ZgWtmef0EvYWUty4+lrZ1IX23rJK/048QVWlfZElK+Mc7THXLQK5Uyfsa6TzGh8lYLgPrirCX81ClwtknRQL7WdE9mwRNeqv1ourAvSWAgkK7fv8aD5RG6+aYw1zzFpzdF6hR1q8RA4qbEHjWHmW0qN8KlRTacqbSCSSiKF+ArkkryoJyBrXC9lnuVhhoJisjdFZwrw1kF3fX8C/ac4FskmeTkNZnjSnisCR15UE+sHs2popC6b+NActEjbql9fgQPY9nOUvJVj4E8Lb1KoEMkjhhasmjxprTULYV5q1wxztBLC+Ndb7U1/m9YS7rcJmhZfzRdiJOLvIe78VzTl5kJwKVqE6mfJjQ2v0HIXMpuY+SzTkSQt9W0Uor0DllaG8QidSx87VG4Hyo1Bgze+WKNJcY8PPpLiuGRkI3ud5zwdLhuWVsbyCR+6/CHeVS6SFXd9tKLatMxlcFJbmCJ6hkjUKBDJIJEieBLAGLPWn01UK10h4U36rjFtCQHVbMJ+W+VgZomGOS3bRM3ReOoJCsIW8sq/20vOsHHWycp1J2P7fiUywvy35cKq0rAxmyYD12uNUsHXXiSS23zYflmQmd96OMXvvRutYMrHQyZvRPjvJfTSn5Y1YxdwnPx9a4sjOeXCiR4/Ni20PW/5O1+b7gOrIlk9lsGTWeiTYqWDr6VTE4FZgM+PqfJGPIhm0DjZLBknZyJNBEJRYu59GSvvrreUB1Duwh/kS7HbwTOrdZ63gs4RXVPoGHKUXJkjw4UI/ztLXAlvHKySy3EgRd4R9LLrxSkkBr5RJwYHA0DuC54D1vBD5IbLIkkEtQsiMFmcr+DUJoSCWDBowz9I9RaBQ7uWybniuBMeWggeDFMkEL0zItbmf4PE0SvBImMvSVdiclOUlgsu1wkwECWFyia4Rwolk+aibaxU3fpB2D4MDhAlwcLDM6CVFXv2g31enG68UWnfPela2+SRTuDOleAnfZscJ1r9uQqiLvGFGCHsgyWutSvDIUbhTHa/wLhOU2gGCp9ckgSxSHWixubSnwsJqgx5aFAj7G4m0bAKLj93P+EVwpBrCbNS+TUQG9TBcWF57dXw/PrRKv0WWihSQQlHGE6Ev0L6Rsco4gCdTh7P/dmHtJYQliGTTGLVeMyLDZMlO9NwB+Nkq/NU89LTa2xrCmCdrF3mGiHSbMfVZAWqlQIhAIBinCGFzGpMrJl31PbnX20VfRIT1XFFbzhBCE+cIY0EE5iIhrKATGUSWjggQ9ZkSwatRwu9glv5Mds1SscLuYoFBfP+aIYQ/yBttsOj7fNoMnmwDeE+ni7N8KaxOc4DIICl0Q3hvH4UbiPhWH10mkGP9BBLXETwkRwjkluZXEZYi+QSFfa1k93UR2k2koizlQ4S5Ki8EFoCKg4HUl9nOnduB6BkUsRXQ3PdSwh7tGimCwCo7zlxnySAp+/JgtMiTKLpf59ZTek+liL8UKL+nkkNFgKS5j/VWKtKu9FliQR4peSXSKmmWJLLnrbFPMlyeDLKeVp3yLplziaDYWs+gJoEkySu5MhDJM+mlzJMit0RKqN6s0typrfNeFtBOBomss+uhvL4lL6oseW85S6bkvYdS830ybMXcbOXj9cgBaPdC6QTpCzpvvetaBDL1dMaYJRVsm+q5dq6wRB+5353IIKW/WYjE1fqqfpB/lh13tq3VB9TXdb8liPSxsHOkJYNs+p2gdkjN707X5PuArU9bX5Y8tWMrf93JYMlZ3Xc2yKCXo2fQdhsdX5boJBjYitWCbpnWAmGzwgJh4a8XigyUIHVh4Ih0kSJaACbn4bkVf3+VLPSIoExBUIo1oEXaaLKQkimix8YwK+SlnPstq7s8efLuiSWTrt0joGrOyZtmNlc3EhZPED4G9jD46mF2DRToxyv3SwRrkvIn99xegidGlRDaozAMuxCUCTHy1ntA5eomKJcDJp+qF7W7lGZN0vKIUFmHTNldAlx2Cws9XfQR9sIRSZeYtC/rh6VyEJIw18jlWMq3PJbklWWtViIR5Mkhy5YWXLsoS0DRYqM2knAm0oCVBt2NFkOEjZK1F4SEqVIBasMwWG6tES4Fk+5Ky9FqrTBQC2lDIDLUXrKCyGNF3is2HEgkiayK8yYtkQ0Kn5Qnj/pL3lqttrN1pD7eDSTlhHK5fR8jWXFTc4/qI8mevZxLV2NudWhoTZBWSJr2w1E7Lmfp1/BeT5o3JCTaEIESwYMOgnVVhLXKLRJXipLybsMi5HWl+rOeQ7ZddE8ZT2LZBV5CfwrQXGaxla5ZRNWGIrcKwPHUv5VOeVW/EamWt+LlLXtWgdNz7Nym69R2lkQU6acwrnoufQiegZbMVr+ulBJGxnrWwjutsiYFSiSj6tCiF0gqRRqj3Wv1vJ3IC6Ub+UREbAesvLGeRVzrhf1Ya3ReiM/PF52Ilfyxc63fq8x5T6d8vq3XQZ5Iw/zu9NkoNjo/dFKybXlE9ljvJcH2A7XxSyWEbN4x35stfz4dK8/mPUx0jSV7MMfsvSpzIfex151um51p5InEjVzbaXzlyZGNlulM1Mep+u9LSa9Tuknu96nGoJUjbfrrzW+dnr2Z9umUv07IX2evz9/Xqd03grNFAtnnv9zko0gGnSZsh+7k2mgn9bxyLeVjmvAGLVlRlgtldmZkkBY4eatIkSkCB47DvQueQKgRNhPuInhCyOIvBXWe9rd+WI+lfoIFXeE3UkxFssgzw5G9rYsggNhNIuXe240nYkTOWDfXZbyVX3UjokD7zUhRcoAbv5axn76YK8ZLjAG78d4ExwnW/QKerKlm9TBI8HwazdJfICyqgvYrEjkgpVV1XyMocztpjwsXidAkvDFFhJkIIHlT7KY9nCu54ec43t/HKH4PInmBaOLoydK+aQfMdre7QMtSNE3w4lC4lzzBRArJG0Hl78qOiRRMCNYr9QspuiLIZMlVSFYP2d5Fi6v015uMZ/XdTyAlRRhWSzC0G3Z1N2k6t0a4NLN8zTcdjfoiO4fDXl0aS/JeEhGZZs+V19EMIexNMflqu17a9/rSOJrOyjVJ8FQaJvQ9EQEDpp4hkBzaR6bQXaRaTbgkCR4gInetVWuIsAm8+pm1llWBZpKwtOvitXFWI4QZag8aeRA28F4nlxHClroJBN1c9hEZJnJNXogam+Ws3CKAtA+FyBcRjGlWZwpnnTF1bz1xZmh/jf2h7Lla/Pswc2R9nulmg5QQxibI6+/HLZhNAyFWz46PEqxZUno0r2psW5d9CQzyjlTfKGbn1HZlQhipyCJ5/2ivLSuQqq9prtJ82QR6KkUuvnyAVfwcJW8EpaVNV0UUq42EAaDYXaK+d4CZZPs3kO6kQJ7qExGxHdBcaD1CrHeL5mydy3uOWKt4JyW9Zb7zn06E0NmGymDDtCRjyOAi+c/mOU8s5JU1e34jynQnBfRUdWTrPX9cBLv2ibP75ai8ktFkmLMeRJtV1PMKdSdFvVOanZRra6RwuWs7EY0QDKcy1knmKZo0LGGUN77m+3W+LbebGNoM4aB6UX/Nh2Raz7KNkDqnIlI2SoDY707pdkrvZOSNxuTJvJ1sveVJP330HDtGbX/rRLi43O+N1mcnYvRU5SR3TafrddzOtRuZV21f3kzY7ZnGy1E+etmQQb/zO7/DpZdeSldXF7fccgv33HPPptMQCQInWhjkOWNh98HJLyB2QF86EEJR7DWyFFvlsCeBYm94Q9IwXnnRQihCqH/nDq7ZW1vbeNFaphWSYBU6eY6obApRkReB8iOiR+XR/j4iPcjyoTh2Ka9SoNT584ROp1h8CG+0Uv1oYVPd1YHyyDClwSEWV9yaEqhrh7I66gZqtSK7q4W1vCiPtew6h2/fGmEDaylaKnuRsN+HBrEIq15CKIjyqXvkuSAyzJJOTWC8Eu5ZrfbjqgOkSy0oBCFOz7ReDH1V6G+172VUJJBd3fjNiRPCHlQKJ7NeQapn5UcClBYTeYcpj7avirixniUpULt4H90u9JES4W1KfVmZunYMkfQMUi4VGElCO68tXMUCPTt2kKwEzyiNidQ8V2NF5S+Za234pRZM63Kt0E2rKEBoR5E3Ip+gfT8Dq6xL4U+cY3oJXBoIqYQgwElok/eWiBx52KguW0AxSejqGWEHwbNPdZ/mrlUeGua/JZ/UD/tN+TW+VHfWU0V1A+2Lun5rHGv+0Vyivq65Qv1V49N6+yivGpOuOkBvrcxqEgR+6+asfbp6CPOGA7q6IU3C/KC0y7S/Fl55k6eUwryUT4VUyuNIeyUpLfW7OoHIsYK6SDXNrTUCQdUAKJRwtR1r16qOFfJg20RklOaRNZQq9PQN01sMnp7bhc1avc4FpTji5DgT8tG5AM3fVYLHsQ2fsd4SJXOunPvkPYdK5p71lLD8Z7sVbDhxjrbltPXSqU7ywn5Ce1ns/05eV53KbJXSfH3aes6nU1gnbdtWlgwqm2sT2vOWb9/88zaK9UiEkym+efIsX/ZOhGSnstt+bfuiLXO+73UiffLX2O9O5N+ZwHrEU75/nU6a63lDnSofGyGEOp3rJAudLO2TjYmTkUYnI1c6raedjuXTX89jMb9Od9LD1kOn+smXPX8+f3y9OcaW137W65tWhtdH5Pd24+UoH70syKA//MM/5CMf+Qif+MQneOCBB3jNa17DW9/6ViYmJjaVTj8h/KIrd24Er6AIzlxjFTdBiloV+L8u8RuiprlrtaGhFkEH7EygPOaV+YuAq/DKvkKhqlk+L7vuWv7pT+7iWAGeJ1imV/BeBQo7kheFFCyrKE3hLdRS0KWU7CAocMvZNQojkTVikbDP0K4sTyIfmgRLhZQpeTfZOkgIb/rpz54pogbznNpV+6hcfBmHp9K2V3/WgCuBK7L2Gb+om9cPlOnL0l7CezXsyq5Ls7Ltyurr0uy82kOK/yuyuhLzPJelN5J9CgTlUJ4EIkN68URNasqxDLymP1i3ZvsvJu0eo3VkBVfOiKzsPnkKyCNiVz/safi2VPuUgX1ZWUaAm7J0B/F9ZhDv+SLF2pl6Vn66CYSBhJWBLP8V8yyyelzIytlDeCPJ+BvfykgLFhP/vO7s+RVgj8r06itJBl9BV1eZq/GbBttY6WK1wtg118CU32Sb7FkiLeTlok2iIWx8nmRlUUiXFP1K1n7q7wo9LOHHrPqq/stbaZQQajZN2GC6lh2Twr4EFNIWTx9ztFI/PkTaiIyTR9zhLP0XCG+2mqX9tbqVpMDQyJW8Ort/h6l77R3UQ/ubcOYJG26KeNGeQxW8t5lCquThJk8chYva+UeLrEhMKRJl8y0SeZVAPA4TyBN58Ng6V0iZ9mBaAVqD+xgf72W24MslMleEzkKW/zEC0QYwNAqNQvueRQuENEYIb1bry+poGt83NZ/J+ywleDrWsnTkibWKb/tZ/By3jO8b8vDqwo8Vza27CP11CUhLVVo7Xk2R8DY+snT7TV2L7Je3pX27GJUaY2NXsrvqvQpfbtgsOfHFL36Rq6++mq6uLq699lq++tWvblNOz3+cKfnobCMhzDuD2WeAEBJuFXER1QpdlievDWfWnCNSqUq74p4nNixJYcmW7UJe+ZQcoo+MXkOEdSjvPbWeZ4ElKGw517vfpiHDkvLRS6hrW882vUrumNqvi/b2GsDPmdoCIU/+VMyzZazTJ+8pdDICJE9SdSJeOhGClpCy5JXWsh6CLKI8Kc82f5Lrax3qSqRQJy+4PGnSiWTKj4s8UbgZoqYTGWLrwOZN9Wo9utZLL5+27rPjcL1+qOd0Kn+n4+sRGJ3ItJORbZ1IN8xxS+adioDrdE7/O5E79jkJwbBnjWf2en1STiQp1iO+OpE6kl3WGwed6rzTJ99P8nnuRPrZPqE5W/rKyxHbLSOVTn3JuY//8l/+C+9973t5z3veA8CnP/1p/uIv/oLPfe5zfPSjH91wOjfcdBP9pRIXERQ1MZU78UL7rLm+l6AY51EgEAVXXwU398NMKxAmsvSnBGXWATsvvZTCa2/lmi4YT7KwHhc2W5bSf+l11zG4Y4zXvWGU5Ra8mvDq6wZ+EZ0lhKRVCEqPFq6F7LxILiluu7M0LiIoh0pjmbB4LeCVmTHC5rBSUMmet0zwnNFEIGKhiVeyVH+6toxXBrVJ7NCOcRJ3JSPX38oVYy368IpfAx8usztLa/e+XvbNNahM1f1C7Hwal2fPOw68Mqu/Hjxh0UXYwFZeGbvxCt6x7DnTWT4vJZAqqoe9hNA861I+gyepBrN87huCm2bAOdj1qleRdA8w/Lqbqc4u0arDNdm1IgqWgUICA9fC1WV4fcYEFLL6HsUruYWsTK/PyqNQmBFCaFR3dr1IsPnsfH92j7wXugneZIv4fiFFHnwoYj3L20qSsPPKqyi+/lZeWYeRNGzWK2FmGhi98joYaVF1F3P5zZO8bnaWBQLR0tvdRc/l17Hr5hVuWArkhiVDerPvGuGV4C7LR192fIYwhgqmf6gfNU1a5ex6CMSQ2njWtK/IM7WL2nkauOwSGJuEqxdCe1cIGxKLmFggLMKl7H61Q0/Whn2lEuOvuJLCrbfyuiwfEEiX3qxsM1medhO8GNVnu0z7d+Png27C/kg7s7q4hLApcR9hDyNLssrrpYifd0QeHifMLRJAdhPCXQuEjcpn8aTWlKm3sez8vle+it07HLeUJ+hvBdJ0ytT1Nfj55+bs2VPA5dmgHG/CLQQST545vdlzpByIfKlm7SGydzzL9wiBUFT5C/g+L685jRuR1XMES/VydrwnqwONjdG+PkaueiWvvfXWtfA5Cf+trM0rWVn7svxonVnO6m14xw7Gr3o1V99ylOp8k4fuvZftgry7NorNWr5ETnz605/mlltu4Td+4zd461vfylNPPcXOnTtPuP773/8+73znO7njjjv4W3/rb/H5z3+ed7zjHTzwwANcc801m3z6hYczJR/ddNNNlEpnT2Qs4Mf3Hvz4reLH5yB+U/kFgpxRy86NEjxlZ7JrtL5oXu4UqgDtSlNeccrfvx3WXylI1ou5H28Y2ouf45fxxiyHnzMla/Zm1w4RwuqX8fPZAuG19/KIFHluyy9Dnw1HkUwqmVIe6lJOrReqy91rvcC10XQXvm1fgW9TpVPEz8NaT2W8GibIBzLg2GeqXa03RL6tpODm29YqpDbv+Xut0lwkbKEwSvBIn83yr/yVs/OXZccr2e85gjGoF79GjeHXmB1ZelME72lo77MWts/m53PbHuvVSyd0Io8sqSJCr5uwZYIMLZ3y2IkM0u8i2ZtFCV7bs4SXxVhPkkLuXkuU5IkQm2dor4f1SMO8N81695PLS75ebfoaU/Jot4SmDF3aJ1JGPRkRJbPL41oekqv4cWD10jyZY/On8ZXPv3Q+3Zuf55SOLb9ti/y5IkEukz4ouUtlsPWZJ64gyKyDhP7QAnY2m3xrG+Uj2JyMdDprw9mQkRLn3PnixdQR9Xqdnp4e/viP/5h3vOMda8dvv/12ZmZm+LM/+7MT7lldXWV1NezEMDc3x549e3jqiSfo7+tb65y2YmQ1t4Mmf00nJEBPGZYb68cO2k41WOuF1UUWG2Ew6g0EurYAlKtV+ostji011xZHG0qjBdk+w+Y3MectAyulVRNVPp8u91uTRiehqEDn+tGkIYHGPlv5tG0wMjxElVWWZpdYbQWhRF4yJTLFsJRQTR3NNAgimkQT/AIiTwftjSLBwdaRJlpN0prMbMiHyqV82jpUP1FIlQP6CrCYPahULjM6PMDKsUnS1NEieBmoDvQ9WoGFetjcFgKpZr3PbLhfQvubrLSwKoTIhlN16s9WGLLni7nzw7VeWFxk0QUSSv1VZRjordBVdKSuxMLCCsvOtS++CezsrbKyuMq8C3m2dVmgc37s4pMfW8pry9yvNODEvq82tsIR5pp8nXWVoNiChmsP4dOzlJYVIpTXVu53Aejv7aW1uLhGQtl9YvJ5tvWs8xozZL+1l49djKUY2PvyQo7ut/nVbxumJtixb++zz7T9D6C3q4tyWme2np4w96h+ZSleMun1FqHRCh5Gepb96Bm2z9jjqj8pGCp7vs9pTlYZlJ6do2z6VnApJAkDXV3MLy+3CUmJSUNp23rVs4tAoVCgt1phdWWF6bl5XvHKVzI7O0t/fz9bhbm5OQYGBk5qwewElXuj+bvlllu46aab+O3f/m0A0jRlz549fPCDH+xITvzDf/gPWVxc5Ctf+crasde//vW89rWv5dOf/vQmcnrh4UzKR0888QR9fX0nXL+dsOSDxo4lcexY0zqZV2byMkmnObDT+XMRqg95mTiC12crd539wIlkVp74Wq+eOqGTEn2yuoYT5xhLEkmugyDLqZ11r1VY1yPmtrPt8mSEPCUg5M32UevlpPVcciB07sPq73mcK300vx6vR6JtJq08ebHR9PJ98VyHLWun8bjeGLPHNlrfG9FddZ3S3Qzy93XKeydZ+1SwJJ/unZmf5+ptkI/g9GSkzcpHcHZkpPPeM+j48eO0Wi3Gxsbajo+NjfHkk092vOeOO+7g3//7f3/C8Z5ajZ4tEnZ683Fn66AJ0NVH1Vy/XkzkEtCby+4GH3Neod5oUqcIPX1rgmA1d402fpYyVexwjd3zqZL7ziN/b/6+04Hd+2N+aRV6a2v/ZVXLYwUoVdtDFDvhbIjoDYBa3wn5LuWuaWSzfVKrtb2KW1gCqFW2fW+UMwG5Lr9UtAD6+tb68Ub62anG+nppnAtzRIsytfUGmUEt97+6zvFzDU2g+yWuJS2gVCtTTf0A2i67TSclbiOYm5tr+1+tVqlW2xu5Xq9z//3387GPfWztWKFQ4LbbbuOuu+7qmO5dd93FRz7ykbZjb33rW/nSl750Grm8sHAm5aNarXbWyaA8CpyZ+fflAEuEb2BqBc7OfhsbhfUIOJfz+VJhDUnnwtocEXE+oeG2Vz6C05ORNiIfwdmTkc57Muh08LGPfayt4vbv389rX/ta9uzZcxZzFRERERER0Rnz8/MMDAxsWfqVSoXx8XGOHDmy6XtrtdoJ6+cnPvEJ/t2/+3dtx06HnDhy5EjH608nnxGnRpSPIiIiIiLOJ2y1fASnLyNtVD6Csycjnfdk0OjoKMVikaNHj7YdP3r0KOPj4x3vyTNyl1xyCQAHDhzY8s50oUEu5gcPHtxyF74LCbFetw6xbrcOsW43D+cc8/Pz7N69tVtJd3V1sX//fur1+qkvzsE5R5K0O053snpFbC+ifHRuI86HW4dYt1uHWLdbg1ivm8d2yUdw+jLS+SAfnfdkUKVS4YYbbuDOO+9ci4lP05Q777yTD3zgAxtKo1Dwkb0DAwNxAG4R+vv7Y91uAWK9bh1i3W4dYt1uDtulhHd1ddHVtXXBCqdDToyPj2/q+oiAKB+dH4jz4dYh1u3WIdbt1iDW6+awnUaKl6uMlH9723mJj3zkI3z2s5/l93//93niiSd43/vex+Li4trbMyIiIiIiIiLOLiw5IYicuPXWWzvec+utt7ZdD/CNb3xj3esj2hHlo4iIiIiIiHMfZ0tGOu89g8DvpH3s2DE+/vGPc+TIEV772tfy9a9//YQYuoiIiIiIiIizh4985CPcfvvt3Hjjjdx88838xm/8Rhs58U/+yT/hoosu4o477gDgQx/6EG9605v4z//5P/P2t7+dL3zhC9x333185jOfOZvFOG8Q5aOIiIiIiIjzA2dDRnpZkEEAH/jABzbs9pxHtVrlE5/4xDkXw/dyQKzbrUGs161DrNutQ6zbiFOREwcOHFgLTQJ4wxvewOc//3l+5Vd+hV/+5V/miiuu4Etf+hLXXHPN2SrCeYcoH52biHW7dYh1u3WIdbs1iPUaAWdHRkrcdr6PLSIiIiIiIiIiIiIiIiIiIiLirOJlsWdQRERERERERERERERERERERMTGEMmgiIiIiIiIiIiIiIiIiIiIiAsIkQyKiIiIiIiIiIiIiIiIiIiIuIAQyaCIiIiIiIiIiIiIiIiIiIiICwgXPBn0O7/zO1x66aV0dXVxyy23cM8995ztLJ3zuOOOO7jpppvo6+tj586dvOMd7+Cpp55qu2ZlZYX3v//9jIyMUKvV+Pt//+9z9OjRtmsOHDjA29/+dnp6eti5cye/9Eu/RLPZ3M6inNP41Kc+RZIkfPjDH147Fuv19HHo0CH+8T/+x4yMjNDd3c21117Lfffdt3beOcfHP/5xdu3aRXd3N7fddhvPPPNMWxpTU1O8613vor+/n8HBQX7hF36BhYWF7S7KOYVWq8Wv/uqvsm/fPrq7u7n88sv5D//hP2DfTRDrNiLi/ESUkTaHKB9tD6J8dGYR5aOtQZSPIs4LuAsYX/jCF1ylUnGf+9zn3GOPPebe+973usHBQXf06NGznbVzGm9961vd7/3e77lHH33UPfjgg+5nfuZn3N69e93CwsLaNb/4i7/o9uzZ4+6880533333ude//vXuDW94w9r5ZrPprrnmGnfbbbe5H/7wh+6rX/2qGx0ddR/72MfORpHOOdxzzz3u0ksvddddd5370Ic+tHY81uvpYWpqyl1yySXu53/+593dd9/tnnvuOfeXf/mX7tlnn1275lOf+pQbGBhwX/rSl9xDDz3k/s7f+Ttu3759bnl5ee2av/k3/6Z7zWte437wgx+47373u+4Vr3iFe+c733k2inTO4JOf/KQbGRlxX/nKV9z+/fvdF7/4RVer1dxv/uZvrl0T6zYi4vxDlJE2jygfbT2ifHRmEeWjrUOUjyLOB1zQZNDNN9/s3v/+96/9b7Vabvfu3e6OO+44i7k6/zAxMeEA953vfMc559zMzIwrl8vui1/84to1TzzxhAPcXXfd5Zxz7qtf/aorFAruyJEja9f87u/+ruvv73erq6vbW4BzDPPz8+6KK65w3/jGN9yb3vSmNWEn1uvp49/8m3/jfuInfmLd82mauvHxcfef/tN/Wjs2MzPjqtWq+1//638555x7/PHHHeDuvffetWu+9rWvuSRJ3KFDh7Yu8+c43v72t7t/+k//aduxv/f3/p5717ve5ZyLdRsRcb4iykgvHVE+OrOI8tGZR5SPtg5RPoo4H3DBhonV63Xuv/9+brvttrVjhUKB2267jbvuuuss5uz8w+zsLADDw8MA3H///TQajba6vfrqq9m7d+9a3d51111ce+21jI2NrV3z1re+lbm5OR577LFtzP25h/e///28/e1vb6s/iPX6UvDlL3+ZG2+8kZ/7uZ9j586dXH/99Xz2s59dO79//36OHDnSVrcDAwPccsstbXU7ODjIjTfeuHbNbbfdRqFQ4O67796+wpxjeMMb3sCdd97J008/DcBDDz3E9773Pd72trcBsW4jIs5HRBnpzCDKR2cWUT4684jy0dYhykcR5wNKZzsDZwvHjx+n1Wq1LQoAY2NjPPnkk2cpV+cf0jTlwx/+MG984xu55pprADhy5AiVSoXBwcG2a8fGxjhy5MjaNZ3qXucuVHzhC1/ggQce4N577z3hXKzX08dzzz3H7/7u7/KRj3yEX/7lX+bee+/lX/yLf0GlUuH2229fq5tOdWfrdufOnW3nS6USw8PDF3TdfvSjH2Vubo6rr76aYrFIq9Xik5/8JO9617sAYt1GRJyHiDLSS0eUj84sony0NYjy0dYhykcR5wMuWDIo4szg/e9/P48++ijf+973znZWznscPHiQD33oQ3zjG9+gq6vrbGfnZYU0Tbnxxhv5tV/7NQCuv/56Hn30UT796U9z++23n+Xcnd/4oz/6I/7gD/6Az3/+87z61a/mwQcf5MMf/jC7d++OdRsREXHBIspHZw5RPto6RPlo6xDlo4jzARdsmNjo6CjFYvGENw0cPXqU8fHxs5Sr8wsf+MAH+MpXvsK3vvUtLr744rXj4+Pj1Ot1ZmZm2q63dTs+Pt6x7nXuQsT999/PxMQEr3vd6yiVSpRKJb7zne/wW7/1W5RKJcbGxmK9niZ27drFq171qrZjr3zlKzlw4AAQ6uZk88H4+DgTExNt55vNJlNTUxd03f7SL/0SH/3oR/lH/+gfce211/Lud7+bf/kv/yV33HEHEOs2IuJ8RJSRXhqifHRmEeWjrUOUj7YOUT6KOB9wwZJBlUqFG264gTvvvHPtWJqm3Hnnndx6661nMWfnPpxzfOADH+B//+//zTe/+U327dvXdv6GG26gXC631e1TTz3FgQMH1ur21ltv5ZFHHmmb4L7xjW/Q399/wqJ0oeDNb34zjzzyCA8++ODa58Ybb+Rd73rX2u9Yr6eHN77xjSe83vfpp5/mkksuAWDfvn2Mj4+31e3c3Bx33313W93OzMxw//33r13zzW9+kzRNueWWW7ahFOcmlpaWKBTal5JisUiapkCs24iI8xFRRjo9RPloaxDlo61DlI+2DlE+ijgvcLZ3sD6b+MIXvuCq1ar77//9v7vHH3/c/bN/9s/c4OBg25sGIk7E+973PjcwMOC+/e1vu8OHD699lpaW1q75xV/8Rbd37173zW9+0913333u1ltvdbfeeuvaeb3i8y1veYt78MEH3de//nW3Y8eOC/4Vn3nYt2U4F+v1dHHPPfe4UqnkPvnJT7pnnnnG/cEf/IHr6elx//N//s+1az71qU+5wcFB92d/9mfu4Ycfdn/37/7djq/3vP76693dd9/tvve977krrrjign+95+233+4uuuiitVen/umf/qkbHR11//pf/+u1a2LdRkScf4gy0uYR5aPtQ5SPzgyifLR1iPJRxPmAC5oMcs65//pf/6vbu3evq1Qq7uabb3Y/+MEPznaWznkAHT+/93u/t3bN8vKy++f//J+7oaEh19PT4372Z3/WHT58uC2d559/3r3tbW9z3d3dbnR01P2rf/WvXKPR2ObSnNvICzuxXk8ff/7nf+6uueYaV61W3dVXX+0+85nPtJ1P09T96q/+qhsbG3PVatW9+c1vdk899VTbNZOTk+6d73ynq9Vqrr+/373nPe9x8/Pz21mMcw5zc3PuQx/6kNu7d6/r6upyl112mfu3//bftr2qN9ZtRMT5iSgjbQ5RPto+RPnozCHKR1uDKB9FnA9InHPu7PgkRURERERERERERERERERERERsNy7YPYMiIiIiIiIiIiIiIiIiIiIiLkREMigiIiIiIiIiIiIiIiIiIiLiAkIkgyIiIiIiIiIiIiIiIiIiIiIuIEQyKCIiIiIiIiIiIiIiIiIiIuICQiSDIiIiIiIiIiIiIiIiIiIiIi4gRDIoIiIiIiIiIiIiIiIiIiIi4gJCJIMiIiIiIiIiIiIiIiIiIiIiLiBEMigiIiIiIiIiIiIiIiIiIiLiAkIkgyIiIjaNn//5n+cd73jH2c5GRERERERERMQ5gygfRUREnE8one0MREREnFtIkuSk5z/xiU/wm7/5mzjntilHERERERERERFnF1E+ioiIeLkhcXHGioiIMDhy5Mja7z/8wz/k4x//OE899dTasVqtRq1WOxtZi4iIiIiIiIg4K4jyUURExMsNMUwsIiKiDePj42ufgYEBkiRpO1ar1U5wg/7Jn/xJPvjBD/LhD3+YoaEhxsbG+OxnP8vi4iLvec976Ovr4xWveAVf+9rX2p716KOP8ra3vY1arcbY2Bjvfve7OX78+DaXOCIiIiIiIiLi5IjyUURExMsNkQyKiIg4I/j93/99RkdHueeee/jgBz/I+973Pn7u536ON7zhDTzwwAO85S1v4d3vfjdLS0sAzMzM8NM//dNcf/313HfffXz961/n6NGj/IN/8A/OckkiIiIiIiIiIs4MonwUERFxriKSQREREWcEr3nNa/iVX/kVrrjiCj72sY/R1dXF6Ogo733ve7niiiv4+Mc/zuTkJA8//DAAv/3bv83111/Pr/3ar3H11Vdz/fXX87nPfY5vfetbPP3002e5NBERERERERERLx1RPoqIiDhXETeQjoiIOCO47rrr1n4Xi0VGRka49tpr146NjY0BMDExAcBDDz3Et771rY7x9T/60Y+48sortzjHERERERERERFbiygfRUREnKuIZFBERMQZQblcbvufJEnbMb2FI01TABYWFvjbf/tv8x//4388Ia1du3ZtYU4jIiIiIiIiIrYHUT6KiIg4VxHJoIiIiLOC173udfzJn/wJl156KaVSnIoiIiIiIiIiIqJ8FBERsV2IewZFREScFbz//e9namqKd77zndx777386Ec/4i//8i95z3veQ6vVOtvZi4iIiIiIiIjYdkT5KCIiYrsQyaCIiIizgt27d/PXf/3XtFot3vKWt3Dttdfy4Q9/mMHBQQqFODVFREREREREXHiI8lFERMR2IXHOubOdiYiIiIiIiIiIiIiIiIiIiIiI7UGklyMiIiIiIiIiIiIiIiIiIiIuIEQyKCIiIiIiIiIiIiIiIiIiIuICQiSDIiIiIiIiIiIiIiIiIiIiIi4gRDIoIiIiIiIiIiIiIiIiIiIi4gJCJIMiIiIiIiIiIiIiIiIiIiIiLiBEMigiIiIiIiIiIiIiIiIiIiLiAkIkgyIiIiIiIiIiIiIiIiIiIiIuIEQyKCIiIiIiIiIiIiIiIiIiIuICQiSDIiIiIiIiIiIiIiIiIiIiIi4gRDIoIiIiIiIiIiIiIiIiIiIi4gJCJIMiIiIiIiIiIiIiIiIiIiIiLiD8/wFR2R6lGC+rowAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "channel = 19\n", - "\n", - "# Get a sample and run through model\n", - "x = data.unsqueeze(0).to(device) # (1, C, F, T)\n", - "with torch.inference_mode():\n", - " x_hat = model(x)\n", - "\n", - "# Move to CPU for plotting\n", - "x_np = x[0, channel].cpu().numpy() # (F, T)\n", - "x_hat_np = x_hat[0, channel].cpu().numpy() # (F, T)\n", - "\n", - "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n", - "\n", - "im0 = axes[0].imshow(x_np, aspect='auto', origin='lower', cmap='gist_heat', vmin=0, vmax=4)\n", - "axes[0].set_title(f'Input (channel {channel})')\n", - "axes[0].set_xlabel('Time')\n", - "axes[0].set_ylabel('Frequency')\n", - "fig.colorbar(im0, ax=axes[0])\n", - "\n", - "im1 = axes[1].imshow(x_hat_np, aspect='auto', origin='lower', cmap='gist_heat', vmin=0, vmax=4)\n", - "axes[1].set_title(f'Reconstruction (channel {channel})')\n", - "axes[1].set_xlabel('Time')\n", - "axes[1].set_ylabel('Frequency')\n", - "fig.colorbar(im1, ax=axes[1])\n", - "\n", - "fig.suptitle(f'{signal_name} — {model_name} autoencoder', fontsize=14)\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 318, - "id": "a0033189", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 48, 128, 977])" - ] - }, - "execution_count": 318, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 319, - "id": "9ce48139", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "torch.Size([16, 12, 16, 123])\n" - ] - } - ], - "source": [ - "with torch.inference_mode():\n", - " z = model.encoder(x)\n", - "\n", - "print(z.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 320, - "id": "9401186b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.06294779938587512\n" - ] - } - ], - "source": [ - "print(z.flatten().shape[0] / x.flatten().shape[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 321, - "id": "48fbf2a0", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": 322, - "id": "0b5c8abd", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "377856" - ] - }, - "execution_count": 322, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "z.flatten().shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 305, - "id": "cbd1ef64", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "np.float64(614.6999267935535)" - ] - }, - "execution_count": 305, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.sqrt(z.flatten().shape[0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fbf35938", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.14" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/scripts/run_demo_2.py b/scripts/run_demo_2.py deleted file mode 100644 index a89e7ba..0000000 --- a/scripts/run_demo_2.py +++ /dev/null @@ -1,127 +0,0 @@ -import numpy as np -from pathlib import Path -import torch -import torch.nn as nn -import torch.optim as optim -from torch.utils.data import DataLoader, ConcatDataset -from torchinfo import summary - -from tokamak_foundation_model.data.data_loader import ( - TokamakH5Dataset, collate_fn, collate_fn_prediction, compute_preprocessing_stats) -from tokamak_foundation_model.models.dummy_model_2 import Fusion4FusionModel, Prediction4FusionModel -from tokamak_foundation_model.models.loss import DictMSELoss -from tokamak_foundation_model.trainer.trainer import Trainer - - -def worker_init_fn(worker_id): - """Each worker needs to open its own file handle.""" - worker_info = torch.utils.data.get_worker_info() - if worker_info is not None: - dataset = worker_info.dataset - # Force re-open file for this worker - if hasattr(dataset, 'datasets'): # ConcatDataset - for ds in dataset.datasets: - ds.h5_file = None - ds._open_hdf5() - else: - dataset.h5_file = None - dataset._open_hdf5() - -print("Initializing and demonstrating custom DataLoader with updated TokamakH5Dataset") -# Use glob to find all generated HDF5 files -hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data").glob("*_processed.h5") - ) - -# Create TokamakH5Dataset instances for each HDF5 file -# datasets = [TokamakH5Dataset(hdf5_path=str(f)) for f in hdf5_files] -# stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') -stats = torch.load('data/preprocessing_stats.pt') - -# All signals the model expects as inputs -all_input_signals = [ - "mhr", "ece", "co2", # spectrograms - "gas", "ech", "pin", "tin", # actuators - "d_alpha", "mse", "ts_core_density", # diagnostics - "bolo", "irtv", "tangtv", # videos - "text", # metadata -] - -datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=all_input_signals, - ) for f in hdf5_files] - -# Concatenate the datasets -concatenated_dataset = ConcatDataset(datasets_processed) - -print(f"Initialized ConcatDataset with {len(concatenated_dataset)} samples.") - -# Initialize DataLoader -dataloader = DataLoader( - concatenated_dataset, - batch_size=2, - shuffle=False, - collate_fn=collate_fn_prediction, - worker_init_fn=worker_init_fn - ) - -# Get and print the first batch from DataLoader to verify functionality -batch = next(iter(dataloader)) # Get the first batch to verify functionality - -# --- 3. Initialize and Demonstrate Dummy PyTorch Model with text input --- -print("\n--- 3. Initializing and demonstrating Dummy PyTorch Model with text input ---") -# Target configs: (n_channels, n_frames) matching dataloader prediction targets -# d_alpha: 6ch, prediction_horizon 0.2s @ 10kHz = 2000 frames -# mse: 69ch, prediction_horizon 0.2s @ 100Hz = 20 frames -# ts_core_density: 44ch, prediction_horizon 0.2s @ 100Hz = 20 frames -target_configs = { - "d_alpha": (6, 2000), - "mse": (69, 20), - "ts_core_density": (44, 20), -} -model = Prediction4FusionModel(target_configs=target_configs) -summary(model, depth=2) - -model.eval() -with torch.no_grad(): - # The batch now includes 'text' data - output = model(batch) -print(f"Model output type: {type(output)}") -for k, v in output.items(): - print(f" {k}: {v.shape}") - -# # --- 4. Initialize and Demonstrate Extensible PyTorch Trainer --- -print("\n--- 4. Initializing and demonstrating Extensible PyTorch Trainer ---") -optimizer = optim.Adam(model.parameters(), lr=0.001) -loss_fn = DictMSELoss() # MSE loss for dict-based outputs -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -model.to(device) -print(f"Using device: {device}") - -trainer = Trainer( - model=model, - optimizer=optimizer, - loss_fn=loss_fn, - device=device, - epochs=10, # Only 1 epoch for demonstration - batch_size=2, - checkpoint_path="dummy_trainer_checkpoint.pth" -) -print("Trainer class initialized.") - -print("Running dummy training epoch...") -# Ensure the model is in training mode before calling _train_epoch -model.train() -train_metrics = trainer.train(dataloader) # Corrected method call -print(f" Finished dummy training epoch. Metrics: {train_metrics}") - -print("Running dummy validation epoch...") -# Ensure the model is in evaluation mode before calling _validate_epoch -model.eval() -val_metrics = trainer._validate_epoch(dataloader) # Corrected method call -print(f" Finished dummy validation epoch. Metrics: {val_metrics}") - -print("\nDemonstration complete!") diff --git a/scripts/run_demo_3.py b/scripts/run_demo_3.py deleted file mode 100644 index 44d9928..0000000 --- a/scripts/run_demo_3.py +++ /dev/null @@ -1,163 +0,0 @@ -import numpy as np -from pathlib import Path -import torch -import torch.optim as optim -from torch.utils.data import DataLoader, ConcatDataset -from dataclasses import dataclass - -from tokamak_foundation_model.data.data_loader import ( - TokamakH5Dataset, collate_fn_prediction, compute_preprocessing_stats) -from tokamak_foundation_model.models.dummy_model_2 import ( - Prediction4FusionModel, DictMSELoss, DEFAULT_MODALITY_CONFIGS) -from tokamak_foundation_model.trainer.trainer import Trainer -from tokamak_foundation_model.models.modality import PROCESSOR_REGISTRY - - -def worker_init_fn(worker_id): - """Each worker needs to open its own file handle.""" - worker_info = torch.utils.data.get_worker_info() - if worker_info is not None: - dataset = worker_info.dataset - if hasattr(dataset, 'datasets'): # ConcatDataset - for ds in dataset.datasets: - ds.h5_file = None - ds._open_hdf5() - else: - dataset.h5_file = None - dataset._open_hdf5() - - -# --- 1. Load data --- -print("--- 1. Loading data ---") -hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data").glob("*_processed.h5") -) -stats = torch.load('data/preprocessing_stats.pt') - -all_input_signals = [ - "mhr", "ece", "co2", - "gas", "ech", "pin", "tin", - "d_alpha", "mse", "ts_core_density", - "bolo", "irtv", "tangtv", - "text", -] - - -datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=all_input_signals, - ) for f in hdf5_files] - -concatenated_dataset = ConcatDataset(datasets_processed) -print(f"ConcatDataset with {len(concatenated_dataset)} samples.") - -dataloader = DataLoader( - concatenated_dataset, - batch_size=1, - shuffle=False, - collate_fn=collate_fn_prediction, - worker_init_fn=worker_init_fn, -) - - -# --- 2. Infer target configs from a sample batch --- -print("\n--- 2. Inferring target configs from sample batch ---") -batch = next(iter(dataloader)) -print("Input keys:", list(batch['inputs'].keys())) -print("Target keys:", list(batch['targets'].keys())) - -target_configs = {} -for name, tensor in batch['targets'].items(): - # targets are (B, C, T) - n_channels = tensor.shape[1] - n_frames = tensor.shape[-1] - target_configs[name] = (n_channels, n_frames) - print(f" target '{name}': channels={n_channels}, frames={n_frames}") - - -# --- 3. Build model --- -print("\n--- 3. Building Prediction4FusionModel ---") - -@dataclass -class ModalityConfig: - name: str - processor_type: str - group: str | None = None - embed_dim: int = 64 - -encoder_modalities = [ - ModalityConfig("mhr", "spectrogram"), - ModalityConfig("ece", "spectrogram"), - ModalityConfig("co2", "spectrogram"), - ModalityConfig("gas", "timeseries", group="actuators"), - ModalityConfig("ech", "timeseries", group="actuators"), - ModalityConfig("pin", "timeseries", group="actuators"), - ModalityConfig("tin", "timeseries", group="actuators"), - ModalityConfig("d_alpha", "fast_timeseries"), - ModalityConfig("mse", "timeseries", group="diagnostics"), - ModalityConfig("ts_core_density", "timeseries", group="diagnostics"), - ModalityConfig("tangtv", "video"), -] - -decoder_modalities = [ - ModalityConfig("d_alpha", "fast_timeseries"), - ModalityConfig("mse", "timeseries", group="diagnostics"), - ModalityConfig("ts_core_density", "timeseries", group="diagnostics"), -] - -encoder_embeddings = {} -decoder_embeddings = {} -global_embeddings = {} - -model = Prediction4FusionModel( - modality_configs=DEFAULT_MODALITY_CONFIGS, - feature_dim=64, - num_heads=4, - target_configs=target_configs, -) -total_params = sum(p.numel() for p in model.parameters()) -trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) -print(f"Total parameters: {total_params:,}") -print(f"Trainable parameters: {trainable_params:,}") - -# --- 4. Forward pass test --- -print("\n--- 4. Forward pass test (no_grad) ---") -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -print(f"Using device: {device}") -model.to(device) - -# Move batch to device -inputs_dev = {k: v.to(device) if isinstance(v, torch.Tensor) else v - for k, v in batch['inputs'].items()} - -model.eval() -with torch.no_grad(): - output = model(inputs_dev) - -print("Prediction shapes:") -for k, v in output.items(): - target_shape = batch['targets'][k].shape - print(f" {k}: pred={v.shape}, target={target_shape}") - -# --- 5. Training test (1 epoch) --- -print("\n--- 5. Training test (1 epoch) ---") -optimizer = optim.Adam(model.parameters(), lr=1e-3) -loss_fn = DictMSELoss() - -trainer = Trainer( - model=model, - optimizer=optimizer, - loss_fn=loss_fn, - device=device, - epochs=1, - batch_size=1, - checkpoint_path="dummy_trainer_checkpoint.pth", -) - -model.train() -train_loss = trainer.train(dataloader) -print(f"Training complete. Final metrics: {train_loss}") - -print("\nDemo complete!") diff --git a/config/shot_list/train_full.txt b/scripts/slurm/train_bes.sh similarity index 100% rename from config/shot_list/train_full.txt rename to scripts/slurm/train_bes.sh diff --git a/scripts/slurm/train_co2.sh b/scripts/slurm/train_co2.sh new file mode 100644 index 0000000..c85388c --- /dev/null +++ b/scripts/slurm/train_co2.sh @@ -0,0 +1,25 @@ +#!/bin/bash +#SBATCH --job-name=train_co2 +#SBATCH --output=logs/%j_train_co2.out +#SBATCH --error=logs/%j_train_co2.err +#SBATCH --time=08:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=2 +#SBATCH --mem-per-cpu=2G + +export OMP_NUM_THREADS=1 +export PYTHONUNBUFFERED=1 + +srun python scripts/train_unimodal_autoencoder.py \ + --signal "co2" \ + --d_model 16 \ + --batch_size 24 \ + --num_workers 2 \ + --epochs 100 \ + --lr 0.001 \ + --n_fft 256 \ + --hop_length 128 \ + --log_interval 5 \ + --checkpoint_dir runs \ No newline at end of file diff --git a/scripts/slurm/train_ece.sh b/scripts/slurm/train_ece.sh new file mode 100644 index 0000000..e374c33 --- /dev/null +++ b/scripts/slurm/train_ece.sh @@ -0,0 +1,28 @@ +#!/bin/bash +#SBATCH --job-name=train_ece +#SBATCH --output=logs/%j_train_ece.out +#SBATCH --error=logs/%j_train_ece.err +#SBATCH --time=08:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=2 +#SBATCH --mem-per-cpu=3G + +export OMP_NUM_THREADS=1 +export PYTHONUNBUFFERED=1 + +srun pixi run python scripts/training/train_unimodal_autoencoder.py \ + --signal ece \ + --data_dir /scratch/gpfs/EKOLEMEN/big_d3d_data/dummy_foundation_model_data \ + --d_model 16 \ + --batch_size 16 \ + --num_workers 8 \ + --epochs 300 \ + --lr 0.001 \ + --n_fft 256 \ + --hop_length 256 \ + --chunk_duration_s 0.05 \ + --log_interval 20 \ + --checkpoint_dir runs \ + # --resume \ No newline at end of file diff --git a/scripts/slurm/train_mhr.sh b/scripts/slurm/train_mhr.sh new file mode 100644 index 0000000..56d5830 --- /dev/null +++ b/scripts/slurm/train_mhr.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --job-name=train_mhr +#SBATCH --output=logs/%j_train_mhr.out +#SBATCH --error=logs/%j_train_mhr.err +#SBATCH --time=08:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem-per-cpu=2G + +export OMP_NUM_THREADS=1 +export PYTHONUNBUFFERED=1 + +srun pixi run python scripts/train_unimodal_autoencoder.py \ + --signal "mhr" \ + --d_model 16 \ + --batch_size 128 \ + --num_workers 4 \ + --epochs 300 \ + --lr 0.001 \ + --n_fft 256 \ + --hop_length 256 \ + --chunk_duration_s 0.05 \ + --log_interval 20 \ + --checkpoint_dir runs \ \ No newline at end of file diff --git a/scripts/slurm/train_unimodal.sh b/scripts/slurm/train_unimodal.sh new file mode 100644 index 0000000..abd316c --- /dev/null +++ b/scripts/slurm/train_unimodal.sh @@ -0,0 +1,25 @@ +#!/bin/bash +#SBATCH --job-name=train_unimodal +#SBATCH --output=logs/%j_train_unimodal.out +#SBATCH --error=logs/%j_train_unimodal.err +#SBATCH --time=04:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem-per-cpu=16G + +export OMP_NUM_THREADS=1 +export PYTHONUNBUFFERED=1 + +srun pixi run python scripts/train_unimodal_autoencoder.py \ + --signal "ece" \ + --d_model 16 \ + --batch_size 3 \ + --num_workers 4 \ + --epochs 200 \ + --lr 0.001 \ + --weight_decay 0.05 \ + --warmup_epochs 5 \ + --min_lr 0.0 \ + --checkpoint_dir runs diff --git a/scripts/train_unimodal.sh b/scripts/train_unimodal.sh deleted file mode 100644 index 66a99a9..0000000 --- a/scripts/train_unimodal.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=train_unimodal -#SBATCH --output=logs/%j_train_unimodal.out -#SBATCH --error=logs/%j_train_unimodal.err -#SBATCH --time=00:30:00 -#SBATCH --nodes=1 -#SBATCH --ntasks-per-node=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem-per-cpu=16G - -export OMP_NUM_THREADS=1 -export PYTHONUNBUFFERED=1 - -EPOCHS=50 -D_MODEL=256 -BATCH_SIZE=2 -NUM_WORKERS=4 -LR=0.001 -WEIGHT_DECAY=0.05 -WARMUP_EPOCHS=5 -MIN_LR=0.0 -CHECKPOINT_DIR=runs - -# All modalities: spectrograms, fast time series, profiles, videos -SIGNALS=(mhr ece co2 d_alpha gas ech pin tin mse ts_core_density bolo irtv tangtv) - -for SIGNAL in "${SIGNALS[@]}"; do - echo "============================================" - echo "Training signal: ${SIGNAL}" - echo "============================================" - srun pixi run python scripts/train_unimodal_autoencoder.py \ - --signal "${SIGNAL}" \ - --d_model "${D_MODEL}" \ - --batch_size "${BATCH_SIZE}" \ - --num_workers "${NUM_WORKERS}" \ - --epochs "${EPOCHS}" \ - --lr "${LR}" \ - --weight_decay "${WEIGHT_DECAY}" \ - --warmup_epochs "${WARMUP_EPOCHS}" \ - --min_lr "${MIN_LR}" \ - --checkpoint_dir "${CHECKPOINT_DIR}" -done - -echo "All modalities complete." diff --git a/scripts/actuator_reconstruction.py b/scripts/training/actuator_reconstruction.py similarity index 100% rename from scripts/actuator_reconstruction.py rename to scripts/training/actuator_reconstruction.py diff --git a/scripts/training/fast_time_series_reconstruction.py b/scripts/training/fast_time_series_reconstruction.py new file mode 100644 index 0000000..808037d --- /dev/null +++ b/scripts/training/fast_time_series_reconstruction.py @@ -0,0 +1,190 @@ +from pathlib import Path +import argparse +import logging + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.data.utils import worker_init_fn +from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) + +from tokamak_foundation_model.utils import DefaultDrawer + + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser.add_argument( + "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + default="d_alpha", + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--hop_length", type=int, default=256, help="Hop length for STFT.", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default="fast_time_series", + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=512, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=140, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=4, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=50, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=5e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.05, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + signal_name = args.signal + model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Signal: {signal_name}, Model: {model_name}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*_processed.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=False, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + # Not sure if this is elegant + sample_data = next(iter(concatenated_dataset))[signal_name] + n_channels = sample_data.shape[0] + logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") + + ### Model Setup ### + model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, + n_channels=n_channels, kernel_size=3).to(device) + + n_params = sum(p.numel() for p in model.parameters()) + logger.info(f"Model parameters: {n_params:,}") + + optimizer = optim.AdamW( + model.parameters(), + lr=args.lr, + ) + + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=args.epochs, + eta_min=args.min_lr + ) + + loss_fn = nn.L1Loss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = UnimodalTrainer( + epochs=args.epochs, + checkpoint_path=checkpoint_path, + model=model, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=checkpoint_path) + + trainer.train(dataloader, modality_key=signal_name) + + +if __name__ == "__main__": + main() diff --git a/scripts/training/profile_reconstruction.py b/scripts/training/profile_reconstruction.py new file mode 100644 index 0000000..91500d9 --- /dev/null +++ b/scripts/training/profile_reconstruction.py @@ -0,0 +1,194 @@ +from pathlib import Path +import argparse +import logging + +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +from tokamak_foundation_model.data.utils import worker_init_fn +from tokamak_foundation_model.trainer.trainer import UnimodalTrainer +from tokamak_foundation_model.models.model_factory import ( + build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) + +from tokamak_foundation_model.utils import DefaultDrawer + + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + + ### Settings ### + parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser.add_argument( + "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), + default="mse", + help="Signal name to train on" + ) + parser.add_argument( + "--n_fft", type=int, default=1024, help="FFT size", + ) + parser.add_argument( + "--hop_length", type=int, default=256, help="Hop length for STFT.", + ) + parser.add_argument( + "--model", choices=list(MODEL_REGISTRY.keys()), default="profile", + help="Model type (default: auto-selected from signal)" + ) + parser.add_argument( + "--data_dir", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + help="Path to HDF5 data directory" + ) + parser.add_argument( + "--stats_path", type=str, + default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + help="Path to preprocessing stats file" + ) + parser.add_argument( + "--d_model", type=int, default=512, help="Model dimension" + ) + parser.add_argument( + "--n_tokens", type=int, default=140, + help="Number of latent tokens (default: use model default)" + ) + parser.add_argument( + "--batch_size", type=int, default=2, + help="Batch size (for spectrograms, each sample's C channels are processed " + "independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", type=int, default=4, help="Number of data loader workers" + ) + parser.add_argument( + "--epochs", type=int, default=50, help="Number of training epochs" + ) + parser.add_argument( + "--lr", type=float, default=5e-3, help="Learning rate" + ) + parser.add_argument( + "--weight_decay", type=float, default=0.01, help="AdamW weight decay" + ) + parser.add_argument( + "--warmup_epochs", type=int, default=5, + help="LR warmup epochs (0 to disable scheduler)" + ) + parser.add_argument( + "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + ) + parser.add_argument( + "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + ) + parser.add_argument( + "--num_plots", type=int, default=4, + help="Number of reconstruction plots per epoch" + ) + parser.add_argument( + "--log_interval", type=int, default=1, help="Plot every N epochs" + ) + parser.add_argument( + "--resume", action="store_true", default=False, + help="Resume training from checkpoint" + ) + args = parser.parse_args() + + ### Paths ### + signal_name = args.signal + model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] + data_dir = Path(args.data_dir) + statistics_path = Path(args.stats_path) + checkpoint_path = ( + Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" + ) + checkpoint_path.parent.mkdir(parents=True, exist_ok=True) + + logger.info(f"Signal: {signal_name}, Model: {model_name}") + + ### Dataset Setup ### + hdf5_files = sorted(data_dir.glob("*_processed.h5")) + stats = torch.load(statistics_path) + + datasets_processed = [ + TokamakH5Dataset( + hdf5_path=str(f), + preprocessing_stats=stats, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + prediction_mode=False, + ) + for f in hdf5_files + ] + + concatenated_dataset = ConcatDataset(datasets_processed) + + # Not sure if this is elegant + sample_data = next(iter(concatenated_dataset))[signal_name] + logger.info(f"Sample data shape: {sample_data.shape}") + n_spatial_points = sample_data.shape[0] + n_time_points = sample_data.shape[1] + logger.info(f"n_spatial_points: {n_spatial_points}, n_time_points: {n_time_points}") + ### Model Setup ### + model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, + n_channels=1, n_spatial_points=n_spatial_points, + n_time_points=n_time_points, kernel_size=3) + + model = model.to(device) + + n_params = sum(p.numel() for p in model.parameters()) + logger.info(f"Model parameters: {n_params:,}") + + optimizer = optim.AdamW( + model.parameters(), + lr=args.lr, + ) + + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, + T_max=args.epochs, + eta_min=args.min_lr + ) + + loss_fn = nn.L1Loss() + + dataloader = DataLoader( + concatenated_dataset, + batch_size=args.batch_size, + collate_fn=collate_fn, + worker_init_fn=worker_init_fn, + num_workers=args.num_workers, + persistent_workers=args.num_workers > 0, + pin_memory=True, + shuffle=True, + ) + + ### Training ### + drawer = DefaultDrawer(num_plots=args.num_plots) + trainer = UnimodalTrainer( + epochs=args.epochs, + checkpoint_path=checkpoint_path, + model=model, + optimizer=optimizer, + lr_scheduler=lr_scheduler, + loss_fn=loss_fn, + device=device, + drawer=drawer, + log_interval=args.log_interval, + ) + + if args.resume and checkpoint_path.exists(): + logger.info(f"Resuming training from checkpoint: {checkpoint_path}") + trainer.load_checkpoint(checkpoint_path=checkpoint_path) + + trainer.train(dataloader, modality_key=signal_name) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_demo.py b/scripts/training/run_demo.py similarity index 100% rename from scripts/run_demo.py rename to scripts/training/run_demo.py diff --git a/scripts/train_unimodal_autoencoder.py b/scripts/training/train_unimodal_autoencoder.py similarity index 91% rename from scripts/train_unimodal_autoencoder.py rename to scripts/training/train_unimodal_autoencoder.py index efd9175..c57618c 100644 --- a/scripts/train_unimodal_autoencoder.py +++ b/scripts/training/train_unimodal_autoencoder.py @@ -115,6 +115,7 @@ def main(): preprocessing_stats=stats, input_signals=[signal_name], target_signals=[signal_name], + chunk_duration_s=args.chunk_duration_s, n_fft=args.n_fft, hop_length=args.hop_length, prediction_mode=False, @@ -123,6 +124,7 @@ def main(): ] concatenated_dataset = ConcatDataset(datasets_processed) + logger.info(f"Concatenated dataset length: {len(concatenated_dataset)}") # Not sure if this is elegant sample_data = next(iter(concatenated_dataset))[signal_name] @@ -138,9 +140,17 @@ def main(): optimizer = optim.AdamW( model.parameters(), lr=args.lr, + weight_decay=args.weight_decay, ) loss_fn = nn.L1Loss() + if args.warmup_epochs > 0: + lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( + optimizer, T_max=args.epochs - args.warmup_epochs, eta_min=args.min_lr + ) + else: + lr_scheduler = optim.lr_scheduler.LRScheduler(optimizer) + dataloader = DataLoader( concatenated_dataset, batch_size=args.batch_size, @@ -153,7 +163,7 @@ def main(): ) ### Training ### - drawer = DefaultDrawer(num_plots=args.num_plots) + drawer = DefaultDrawer(num_plots=args.num_plots) # TODO: make more consistent trainer = UnimodalTrainer( epochs=args.epochs, checkpoint_path=checkpoint_path, @@ -162,6 +172,7 @@ def main(): loss_fn=loss_fn, device=device, drawer=drawer, + lr_scheduler=lr_scheduler, log_interval=args.log_interval, ) diff --git a/scripts/video_reconstruction.py b/scripts/training/video_reconstruction.py similarity index 100% rename from scripts/video_reconstruction.py rename to scripts/training/video_reconstruction.py diff --git a/src/tokamak_foundation_model/data/config/config.yaml b/src/tokamak_foundation_model/data/config/config.yaml new file mode 100644 index 0000000..b8266b3 --- /dev/null +++ b/src/tokamak_foundation_model/data/config/config.yaml @@ -0,0 +1,7 @@ +defaults: + - modalities: modalities + - shot_list: train_small + +# These can be overridden from CLI, e.g.: +# python generate_data.py shot_list=train +# python generate_data.py modalities.input_data_path=/other/path \ No newline at end of file diff --git a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml new file mode 100644 index 0000000..caa712e --- /dev/null +++ b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml @@ -0,0 +1,138 @@ +# Modality definitions for data processing +# Each modality specifies how to read from the input HDF5 and write to output + +input_data_path: /scratch/gpfs/EKOLEMEN/d3d_fusion_data +output_data_path: /scratch/gpfs/EKOLEMEN/foundation_model + +# TODO: merge video data into input_data_path, then remove this +video_data_path: /scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_image_data + +num_workers: 64 + +signals: + bes: + input_group: bes + input_xkey: axis1 + input_ykey: block0_values + source: default # reads from {shot}.h5 + stft: true + sampling_rate: 500000 + num_channels: 64 + + dalpha: + input_group: d_alpha + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: true + sampling_rate: 500000 + num_channels: 16 + + mse: + input_group: mse + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: false + sampling_rate: 1000 + num_channels: 36 + + ts_core_density: + input_group: ts_core_density + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: false + sampling_rate: 1000 + num_channels: 40 + + mhr: + input_group: magnetics_high_resolution + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: true + sampling_rate: 500000 + num_channels: 8 + + ece: + input_group: ece_cali + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: true + sampling_rate: 500000 + num_channels: 48 + + co2: + input_group: co2_density + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: true + sampling_rate: 500000 + num_channels: 4 + + gas: + input_group: gas + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: false + sampling_rate: 1000 + num_channels: 5 + + ech: + input_group: ech + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: false + sampling_rate: 1000 + num_channels: 11 + + pin: + input_group: p_inj + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: false + sampling_rate: 1000 + num_channels: 8 + + tin: + input_group: t_inj + input_xkey: axis1 + input_ykey: block0_values + source: default + stft: false + sampling_rate: 1000 + num_channels: 8 + + bolo: + input_group: bolo + input_xkey: time + input_ykey: data + source: video # reads from video_data_path/{shot}_image.h5 + stft: false + sampling_rate: 1000 + num_channels: 48 + # swap_axes: [0, 2] # swapaxes on ydata + + irtv: + input_group: irtv + input_xkey: time + input_ykey: data + source: video + stft: false + sampling_rate: 1000 + num_channels: 48 + + tangtv: + input_group: tangtv + input_xkey: time + input_ykey: data + source: video + stft: false + sampling_rate: 1000 + num_channels: 48 \ No newline at end of file diff --git a/src/tokamak_foundation_model/data/config/shot_list/train_debug.yaml b/src/tokamak_foundation_model/data/config/shot_list/train_debug.yaml new file mode 100644 index 0000000..5d18c81 --- /dev/null +++ b/src/tokamak_foundation_model/data/config/shot_list/train_debug.yaml @@ -0,0 +1,11 @@ +# Small shot list for debugging / quick iteration +shots: + - 182620 + - 182671 + - 189262 + - 189285 + - 191726 + - 192012 + - 192248 + - 195078 + - 196026 \ No newline at end of file diff --git a/config/shot_list/train_medium.txt b/src/tokamak_foundation_model/data/config/shot_list/train_full.txt similarity index 100% rename from config/shot_list/train_medium.txt rename to src/tokamak_foundation_model/data/config/shot_list/train_full.txt diff --git a/config/shot_list/train_small.txt b/src/tokamak_foundation_model/data/config/shot_list/train_medium.txt similarity index 100% rename from config/shot_list/train_small.txt rename to src/tokamak_foundation_model/data/config/shot_list/train_medium.txt diff --git a/src/tokamak_foundation_model/data/config/shot_list/train_small.yaml b/src/tokamak_foundation_model/data/config/shot_list/train_small.yaml new file mode 100644 index 0000000..7d904ac --- /dev/null +++ b/src/tokamak_foundation_model/data/config/shot_list/train_small.yaml @@ -0,0 +1,640 @@ +# Training shot list (638 unique shots) +shots: + - 155541 + - 159300 + - 161172 + - 162975 + - 163303 + - 164388 + - 164394 + - 164397 + - 164399 + - 164409 + - 165381 + - 165383 + - 169532 + - 170000 + - 170008 + - 170659 + - 170660 + - 170661 + - 170662 + - 170663 + - 170664 + - 170665 + - 170666 + - 170667 + - 170669 + - 170670 + - 170671 + - 170672 + - 170675 + - 170677 + - 170678 + - 170679 + - 170714 + - 170715 + - 170716 + - 170717 + - 170718 + - 170719 + - 170720 + - 170721 + - 170722 + - 170724 + - 170725 + - 170727 + - 170729 + - 170730 + - 170790 + - 170791 + - 170792 + - 170793 + - 170794 + - 170795 + - 170797 + - 170798 + - 170799 + - 170801 + - 170803 + - 170805 + - 170806 + - 170807 + - 170808 + - 170809 + - 170810 + - 170811 + - 170813 + - 170814 + - 170815 + - 170816 + - 171322 + - 171998 + - 171999 + - 172000 + - 172001 + - 172002 + - 172005 + - 172007 + - 172008 + - 172009 + - 172010 + - 172011 + - 172012 + - 172013 + - 172017 + - 172018 + - 172019 + - 172024 + - 172025 + - 172028 + - 172325 + - 172330 + - 174082 + - 175239 + - 175240 + - 175241 + - 175244 + - 175245 + - 175250 + - 175251 + - 175252 + - 175955 + - 175956 + - 175957 + - 175958 + - 175960 + - 175962 + - 175963 + - 175964 + - 175965 + - 175967 + - 175970 + - 175971 + - 175972 + - 175973 + - 175975 + - 175976 + - 175977 + - 175978 + - 175979 + - 175980 + - 175981 + - 175982 + - 175983 + - 175984 + - 175985 + - 175986 + - 175987 + - 176003 + - 176005 + - 176006 + - 176007 + - 176009 + - 176010 + - 176011 + - 176012 + - 176013 + - 176014 + - 176031 + - 176032 + - 176033 + - 176035 + - 176036 + - 176037 + - 176038 + - 176039 + - 176040 + - 176041 + - 176042 + - 176043 + - 176044 + - 176045 + - 176046 + - 176047 + - 176048 + - 176049 + - 176050 + - 176052 + - 176053 + - 176054 + - 176055 + - 176056 + - 176057 + - 176058 + - 176060 + - 176061 + - 176517 + - 176518 + - 176519 + - 176520 + - 176521 + - 176522 + - 176524 + - 176526 + - 176528 + - 176529 + - 176531 + - 176532 + - 176533 + - 176537 + - 176540 + - 176541 + - 176543 + - 176544 + - 176547 + - 176548 + - 176549 + - 176551 + - 178630 + - 178631 + - 178632 + - 178633 + - 178634 + - 178635 + - 178636 + - 178637 + - 178639 + - 178640 + - 178641 + - 178642 + - 178786 + - 178787 + - 178828 + - 178829 + - 178868 + - 178984 + - 178985 + - 178986 + - 178987 + - 178988 + - 178994 + - 178995 + - 179064 + - 179065 + - 179066 + - 179067 + - 179068 + - 179069 + - 179070 + - 179071 + - 179072 + - 179073 + - 179074 + - 179075 + - 179076 + - 179077 + - 179078 + - 179079 + - 179129 + - 179130 + - 179131 + - 179159 + - 179163 + - 180257 + - 180636 + - 182357 + - 182614 + - 182616 + - 182617 + - 182618 + - 182620 + - 182635 + - 182636 + - 182639 + - 182640 + - 182641 + - 182642 + - 182643 + - 182644 + - 182645 + - 182665 + - 182669 + - 182670 + - 182671 + - 182672 + - 182673 + - 182674 + - 182675 + - 182676 + - 182677 + - 182678 + - 182679 + - 182684 + - 182685 + - 182701 + - 182702 + - 182703 + - 182704 + - 182705 + - 185813 + - 185814 + - 185815 + - 185816 + - 185818 + - 185819 + - 185820 + - 185821 + - 185822 + - 185824 + - 185825 + - 185826 + - 185827 + - 185832 + - 185834 + - 185835 + - 185836 + - 185838 + - 185839 + - 185840 + - 185841 + - 185844 + - 185845 + - 185846 + - 185847 + - 186093 + - 186112 + - 186113 + - 186114 + - 186115 + - 186116 + - 186117 + - 186118 + - 186119 + - 186120 + - 186121 + - 186126 + - 186128 + - 186129 + - 186130 + - 186133 + - 186134 + - 186135 + - 186210 + - 186222 + - 186223 + - 186224 + - 186225 + - 186226 + - 186228 + - 186251 + - 186252 + - 186253 + - 186254 + - 186472 + - 186473 + - 187076 + - 187179 + - 187180 + - 187181 + - 187185 + - 188976 + - 188977 + - 189100 + - 189157 + - 189158 + - 189159 + - 189160 + - 189161 + - 189162 + - 189163 + - 189164 + - 189165 + - 189166 + - 189167 + - 189261 + - 189262 + - 189263 + - 189264 + - 189265 + - 189266 + - 189267 + - 189268 + - 189270 + - 189271 + - 189272 + - 189273 + - 189274 + - 189275 + - 189276 + - 189277 + - 189284 + - 189285 + - 189286 + - 189287 + - 189306 + - 189307 + - 189308 + - 189309 + - 189310 + - 189311 + - 189312 + - 189313 + - 189315 + - 189316 + - 189331 + - 189332 + - 189333 + - 189334 + - 189409 + - 189428 + - 189429 + - 189430 + - 189442 + - 189443 + - 189444 + - 189445 + - 189448 + - 189451 + - 189548 + - 189549 + - 189550 + - 189551 + - 189552 + - 189553 + - 189554 + - 189555 + - 189556 + - 189557 + - 189558 + - 189559 + - 189560 + - 189561 + - 189562 + - 189563 + - 189596 + - 189597 + - 189598 + - 189599 + - 189601 + - 189602 + - 189603 + - 189604 + - 189605 + - 189606 + - 189607 + - 189609 + - 189610 + - 189611 + - 189630 + - 189631 + - 189632 + - 189634 + - 189635 + - 189636 + - 189637 + - 189638 + - 189639 + - 189640 + - 189641 + - 189642 + - 189644 + - 189652 + - 189673 + - 189674 + - 189676 + - 189677 + - 189678 + - 189679 + - 189680 + - 189681 + - 189682 + - 189683 + - 189684 + - 189685 + - 189686 + - 189698 + - 189699 + - 189700 + - 189701 + - 189702 + - 189723 + - 189724 + - 189725 + - 189726 + - 190109 + - 190110 + - 190115 + - 190116 + - 191652 + - 191653 + - 191654 + - 191655 + - 191686 + - 191726 + - 191727 + - 191729 + - 191739 + - 191740 + - 191741 + - 191742 + - 191743 + - 191744 + - 191745 + - 191746 + - 191747 + - 191748 + - 191749 + - 191771 + - 191945 + - 191946 + - 191947 + - 191948 + - 191949 + - 191950 + - 191951 + - 191952 + - 191953 + - 191954 + - 191955 + - 191965 + - 191966 + - 191967 + - 191968 + - 192006 + - 192007 + - 192008 + - 192009 + - 192010 + - 192011 + - 192012 + - 192013 + - 192014 + - 192090 + - 192091 + - 192092 + - 192248 + - 192249 + - 192250 + - 192252 + - 192302 + - 192303 + - 192305 + - 192307 + - 192309 + - 192310 + - 192311 + - 192394 + - 192395 + - 192396 + - 192397 + - 192398 + - 192399 + - 192400 + - 192401 + - 192404 + - 192410 + - 192411 + - 192412 + - 192413 + - 192414 + - 192417 + - 192704 + - 192705 + - 192706 + - 192780 + - 192781 + - 192782 + - 192783 + - 192784 + - 192785 + - 192786 + - 192787 + - 192788 + - 192789 + - 192790 + - 192792 + - 192793 + - 192906 + - 192912 + - 192913 + - 192926 + - 192934 + - 192935 + - 192936 + - 192937 + - 192938 + - 193090 + - 193091 + - 193092 + - 193093 + - 193094 + - 193095 + - 193096 + - 193097 + - 193125 + - 193126 + - 193127 + - 193128 + - 193129 + - 193130 + - 193131 + - 193132 + - 193138 + - 193139 + - 193140 + - 193141 + - 193142 + - 193207 + - 193208 + - 193211 + - 193282 + - 193593 + - 195049 + - 195050 + - 195051 + - 195052 + - 195053 + - 195054 + - 195055 + - 195056 + - 195057 + - 195058 + - 195059 + - 195060 + - 195061 + - 195062 + - 195063 + - 195064 + - 195065 + - 195066 + - 195067 + - 195068 + - 195070 + - 195071 + - 195072 + - 195073 + - 195074 + - 195075 + - 195076 + - 195077 + - 195078 + - 195079 + - 195080 + - 195081 + - 195187 + - 195496 + - 195969 + - 196018 + - 196026 + - 196531 + - 196532 + - 196533 + - 196534 + - 196535 + - 196536 + - 196537 + - 196538 + - 196541 + - 196542 + - 196561 + - 196562 + - 196565 + - 196566 + - 196567 + - 199749 + - 200199 + - 200563 + - 200682 + - 201927 + - 203692 \ No newline at end of file diff --git a/config/shot_list/validation.txt b/src/tokamak_foundation_model/data/config/shot_list/validation.txt similarity index 100% rename from config/shot_list/validation.txt rename to src/tokamak_foundation_model/data/config/shot_list/validation.txt diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index 433cf8b..10045b2 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -9,6 +9,42 @@ import copy +# TODO: implement this for calculation +class Welford: + def __init__(self): + self.mean = 0 + self.std = 0 + self.min_val = 0 + self.max_val = 0 + self.n = 0 + self.M2 = 0 + + def update(self, value): + + if np.isnan(value): + return + + self.n += 1 + delta = value - self.mean + self.mean += delta / self.n + delta2 = value - self.mean + self.M2 += delta * delta2 + self.min_val = min(self.min_val, value) + self.max_val = max(self.max_val, value) + + def _compute_std(self): + self.std = np.sqrt(self.M2 / (self.n - 1 + 1e-8)) + + def compute(self): + self._compute_std() + return { + "mean": self.mean, + "std": self.std, + "min_val": self.min_val, + "max_val": self.max_val, + } + + def compute_preprocessing_stats( datasets, output_path="preprocessing_stats.pt", num_samples=1000 ): @@ -164,7 +200,7 @@ class TokamakH5Dataset(Dataset): 4, 500e3, apply_stft=True, - preprocess=PreprocessConfig(method="log_standardize"), + preprocess=PreprocessConfig(method="log"), ), SignalConfig( "d_alpha", @@ -436,7 +472,7 @@ def _load_signal_raw( duration_s = t_end - t_start ydata = np.zeros( - (round(duration_s * fs_raw), config.num_channels), dtype=np.float32 + (max(1, round(duration_s * fs_raw)), config.num_channels), dtype=np.float32 ) start_idx = max(0, int((t_start - t0) * fs_raw)) diff --git a/src/tokamak_foundation_model/data/dummy_data.py b/src/tokamak_foundation_model/data/dummy_data.py index 2453dbe..983c833 100644 --- a/src/tokamak_foundation_model/data/dummy_data.py +++ b/src/tokamak_foundation_model/data/dummy_data.py @@ -162,7 +162,7 @@ def create_multi_sample_hdf5( def create_single_sample_hdf5(): - data_path = Path("C:\\Temp") + data_path = Path("/scratch/gpfs/EKOLEMEN/d3d_fusion_data") shot = 182620 with h5py.File(data_path / f"{shot}.h5", "r") as f: bes = ( @@ -224,8 +224,8 @@ def create_single_sample_hdf5(): f["tangtv"]["data"][:], ) - with open(data_path / f"{shot}.txt", "r") as f: - logfile = f.read() + # with open(data_path / f"{shot}.txt", "r") as f: + # logfile = f.read() with h5py.File(data_path / f"{shot}_processed.h5", "w") as f: signal_group = f.create_group("bes") @@ -271,6 +271,6 @@ def create_single_sample_hdf5(): signal_group.create_dataset("xdata", data=tangtv[0]) signal_group.create_dataset("ydata", data=tangtv[1]) signal_group = f.create_group("log") - signal_group.create_dataset( - "data", data=np.array(logfile, dtype=h5py.string_dtype(encoding="utf-8")) - ) + # signal_group.create_dataset( + # "data", data=np.array(logfile, dtype=h5py.string_dtype(encoding="utf-8")) + # ) diff --git a/src/tokamak_foundation_model/data/prepare_data.py b/src/tokamak_foundation_model/data/prepare_data.py new file mode 100644 index 0000000..892c47c --- /dev/null +++ b/src/tokamak_foundation_model/data/prepare_data.py @@ -0,0 +1,162 @@ +import numpy as np +import h5py +import hydra +import logging +from multiprocessing import Pool +from functools import partial +from omegaconf import DictConfig, OmegaConf +from pathlib import Path +from tqdm.auto import tqdm + +log = logging.getLogger(__name__) + +# ── hardcoded until video data is merged into the main data path ── +_VIDEO_DATA_PATH = Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_image_data") + + +def _get_valid_shots( + shot_list: list[int], + input_data_path: Path, + video_data_path: Path, +) -> list[int]: + """Return only shots that have files in *both* the main data path and the + video data path. Expects ``{shot}.h5`` in input_data_path and + ``{shot}_image.h5`` in video_data_path.""" + + main_shots = { + int(p.stem) + for p in input_data_path.glob("*.h5") + if p.stem.isdigit() + } + video_shots = { + int(p.stem.replace("_image", "")) + for p in video_data_path.glob("*_image.h5") + } + available = main_shots & video_shots + requested = set(shot_list) + valid = sorted(requested & available) + + n_missing = len(requested) - len(valid) + if n_missing: + log.warning( + f"{n_missing}/{len(requested)} requested shots missing from one or " + f"both data paths – skipped" + ) + log.info(f"{len(valid)} shots available in both paths") + return valid + + +def _process_shot(shot: int, cfg_dict: dict) -> str | None: + """Worker function executed in a child process. + + Args: + shot: Shot number. + cfg_dict: Plain dict (not DictConfig – must be picklable). + + Returns: + None on success, or an error message string on failure. + """ + try: + input_data_path = Path(cfg_dict["input_data_path"]) + video_data_path = Path(cfg_dict.get("video_data_path", str(_VIDEO_DATA_PATH))) + output_data_path = Path(cfg_dict["output_data_path"]) + output_data_path.mkdir(parents=True, exist_ok=True) + + output_file = output_data_path / f"{shot}_processed.h5" + + signals = cfg_dict["signals"] + + # ── group signals by source ── + source_to_signals: dict[str, list[tuple[str, dict]]] = {} + for abbr, sig_cfg in signals.items(): + source = sig_cfg.get("source", "default") + source_to_signals.setdefault(source, []).append((abbr, sig_cfg)) + + # Map source key → input filename + source_file_map = { + "default": input_data_path / f"{shot}.h5", + "video": video_data_path / f"{shot}_image.h5", + } + + # ── read all signals ── + read_data: dict[str, tuple[np.ndarray, np.ndarray]] = {} + + for source_key, sigs in source_to_signals.items(): + fpath = source_file_map.get(source_key) + if fpath is None or not fpath.exists(): + continue + + with h5py.File(fpath, "r") as f: + for abbr, sig_cfg in sigs: + grp_name = sig_cfg["input_group"] + if grp_name not in f: + continue + + xdata = f[grp_name][sig_cfg["input_xkey"]][:] + ydata = f[grp_name][sig_cfg["input_ykey"]][:] + + if sig_cfg.get("swap_axes") is not None: + ydata = ydata.swapaxes(*sig_cfg["swap_axes"]) + + read_data[abbr] = (xdata, ydata) + + if not read_data: + return f"shot {shot}: no data read – skipped" + + # ── write processed file ── + with h5py.File(output_file, "w") as f: + for abbr, (xdata, ydata) in read_data.items(): + grp = f.create_group(abbr) + grp.create_dataset("xdata", data=xdata) + grp.create_dataset("ydata", data=ydata) + + return None # success + + except Exception as e: + return f"shot {shot}: {type(e).__name__}: {e}" + + +@hydra.main(version_base=None, config_path="config", config_name="config") +def main(cfg: DictConfig) -> None: + log.info(f"Config:\n{OmegaConf.to_yaml(cfg)}") + + mod_cfg = cfg.modalities + input_data_path = Path(mod_cfg.input_data_path) + video_data_path = Path(mod_cfg.get("video_data_path", str(_VIDEO_DATA_PATH))) + num_workers = mod_cfg.get("num_workers", 8) + + # ── filter to shots that exist in both paths ── + shots = _get_valid_shots( + shot_list=list(cfg.shot_list.shots), + input_data_path=input_data_path, + video_data_path=video_data_path, + ) + + if not shots: + log.error("No valid shots found – exiting.") + return + + # Convert to plain dict so it's picklable for multiprocessing + cfg_dict = OmegaConf.to_container(mod_cfg, resolve=True) + + log.info(f"Processing {len(shots)} shots with {num_workers} workers") + + worker = partial(_process_shot, cfg_dict=cfg_dict) + + errors = [] + + with Pool(processes=num_workers) as pool: + for i, err in enumerate(tqdm(pool.imap_unordered(worker, shots), total=len(shots))): + if err is not None: + log.error(err) + errors.append(err) + + log.info( + f"Done. {len(shots) - len(errors)}/{len(shots)} succeeded, " + f"{len(errors)} failed." + ) + + +if __name__ == "__main__": + # python -m tokamak_foundation_model.data.prepare_data + main() \ No newline at end of file diff --git a/src/tokamak_foundation_model/models/modality/cer_model.py b/src/tokamak_foundation_model/models/modality/cer_model.py new file mode 100644 index 0000000..2a595e0 --- /dev/null +++ b/src/tokamak_foundation_model/models/modality/cer_model.py @@ -0,0 +1,84 @@ +import torch +import torch.nn as nn + + +class ResidualBlock(nn.Module): + def __init__(self, in_channels, out_channels, kernel_size=3, bias=True): + super(ResidualBlock, self).__init__() + if isinstance(kernel_size, tuple): + padding = tuple(ks // 2 for ks in kernel_size) + else: + padding = kernel_size // 2 + + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, + padding=padding, bias=bias) + self.batch_norm_1 = nn.BatchNorm2d(out_channels) + self.relu = nn.LeakyReLU(negative_slope=0.1, inplace=True) + self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=kernel_size, + padding=padding, bias=bias) + self.batch_norm_2 = nn.BatchNorm2d(out_channels) + + if in_channels != out_channels: + self.skip_conv = nn.Conv2d(in_channels, out_channels, kernel_size=1, + padding=0, bias=bias) + else: + self.skip_conv = None + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.batch_norm_1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.batch_norm_2(out) + if self.skip_conv is not None: + residual = self.skip_conv(residual) + out += residual + out = self.relu(out) + return out + + +class Encoder(nn.Module): + def __init__(self, input_channels, kernel_size=3, bias=True, dropout=0.1): + super(Encoder, self).__init__() + + self.encoder = nn.Sequential( + ResidualBlock(in_channels=input_channels, out_channels=128, + kernel_size=kernel_size, bias=bias), + nn.Dropout(p=dropout), + nn.MaxPool2d(kernel_size=(3, 2), stride=(1, 2), padding=(3 // 2, 0)), + + ResidualBlock(in_channels=128, out_channels=256, + kernel_size=kernel_size, bias=bias), + nn.Dropout(p=dropout), + nn.MaxPool2d(kernel_size=(3, 2), stride=(1, 2), padding=(3 // 2, 0)), + + ResidualBlock(in_channels=256, out_channels=256, + kernel_size=kernel_size, bias=bias), + nn.Dropout(p=dropout), + nn.MaxPool2d(kernel_size=(3, 2), stride=(1, 2), padding=(3 // 2, 0)), + + ResidualBlock(in_channels=256, out_channels=128, + kernel_size=kernel_size, bias=bias), + nn.Dropout(p=dropout), + nn.MaxPool2d(kernel_size=(3, 2), stride=(1, 2), padding=(3 // 2, 0)), + + ResidualBlock(in_channels=128, out_channels=input_channels, + kernel_size=kernel_size, bias=bias), + nn.Dropout(p=dropout), + nn.MaxPool2d(kernel_size=(3, 2), stride=(1, 2), padding=(3 // 2, 0)), + ) + + def forward(self, x): + return self.encoder(x) + + +if __name__ == "__main__": + # python -m tokamak_foundation_model.models.modality.cer_model + encoder = Encoder(input_channels=80, kernel_size=3, bias=True, dropout=0.1) + x = torch.randn(2, 80, 256, 530) + with torch.inference_mode(): + y = encoder(x) + print(y.shape) + + print(f"Compression ratio: {x.numel() / y.numel()}") \ No newline at end of file diff --git a/src/tokamak_foundation_model/models/modality/spectrogram_baseline.py b/src/tokamak_foundation_model/models/modality/spectrogram_baseline.py index 4eaef36..22c002e 100644 --- a/src/tokamak_foundation_model/models/modality/spectrogram_baseline.py +++ b/src/tokamak_foundation_model/models/modality/spectrogram_baseline.py @@ -5,6 +5,39 @@ from .base import ModalityEncoder, ModalityDecoder, ModalityAutoEncoder +class ResBlock3d(nn.Module): + def __init__(self, channels, bottleneck=32): + super().__init__() + self.block = nn.Sequential( + nn.Conv3d(channels, bottleneck, kernel_size=1), # squeeze + nn.BatchNorm3d(bottleneck), + nn.GELU(), + nn.Conv3d(bottleneck, bottleneck, kernel_size=3, padding=1), # cheap 3x3 + nn.BatchNorm3d(bottleneck), + nn.GELU(), + nn.Conv3d(bottleneck, channels, kernel_size=1), # expand + nn.BatchNorm3d(channels), + ) + self.act = nn.GELU() + + def forward(self, x): + return self.act(x + self.block(x)) + + +class TemporalLSTM(nn.Module): + """LSTM along the time dimension of a 5D tensor (B, C, D, H, T).""" + def __init__(self, channels: int, num_layers: int = 1): + super().__init__() + self.lstm = nn.LSTM(channels, channels, num_layers=num_layers, batch_first=True) + + def forward(self, x): + B, C, D, H, T = x.shape + x = x.permute(0, 2, 3, 4, 1).reshape(B * D * H, T, C) + x, _ = self.lstm(x) + x = x.reshape(B, D, H, T, C).permute(0, 4, 1, 2, 3) + return x + + class SpectrogramBaselineEncoder(ModalityEncoder): def __init__(self, n_channels: int, @@ -17,12 +50,18 @@ def __init__(self, self.net = nn.Sequential( nn.Conv3d(dims[0], dims[1], kernel_size=3, padding=1), + nn.BatchNorm3d(dims[1]), nn.GELU(), nn.Conv3d(dims[1], dims[2], kernel_size=3, stride=(1, 2, 2), padding=1), + nn.BatchNorm3d(dims[2]), nn.GELU(), nn.Conv3d(dims[2], dims[3], kernel_size=3, stride=2, padding=1), + nn.BatchNorm3d(dims[3]), nn.GELU(), + ResBlock3d(dims[3]), + TemporalLSTM(dims[3]), nn.Conv3d(dims[3], dims[4], kernel_size=3, stride=2, padding=1), + nn.BatchNorm3d(dims[4]), nn.GELU(), ) @@ -45,28 +84,34 @@ def __init__(self, self.net = nn.Sequential( nn.Upsample(scale_factor=2, mode="trilinear", align_corners=False), nn.Conv3d(dims[4], dims[3], kernel_size=3, padding=1), + nn.BatchNorm3d(dims[3]), nn.GELU(), + TemporalLSTM(dims[3]), + ResBlock3d(dims[3]), nn.Upsample(scale_factor=2, mode="trilinear", align_corners=False), nn.Conv3d(dims[3], dims[2], kernel_size=3, padding=1), + nn.BatchNorm3d(dims[2]), nn.GELU(), nn.Upsample(scale_factor=(1, 2, 2), mode="trilinear", align_corners=False), nn.Conv3d(dims[2], dims[1], kernel_size=3, padding=1), + nn.BatchNorm3d(dims[1]), nn.GELU(), nn.Conv3d(dims[1], dims[0], kernel_size=3, padding=1), ) def forward(self, z, output_shape=None): - x = self.net(z) + y = self.net(z) if output_shape is not None: - x = F.interpolate( - x, size=output_shape, mode="trilinear", align_corners=False + y = F.interpolate( + y, size=output_shape, mode="trilinear", align_corners=False ) - x = x.squeeze(1) - return x + y = y.squeeze(1) + return y class SpectrogramBaselineAutoEncoder(ModalityAutoEncoder): """ Based on 3DCAE implementation at https://github.com/micah35s/Autoencoder-Image-Compression + https://github.com/faadi809/HSI-compression-benchmark """ def __init__(self, @@ -118,10 +163,10 @@ def _run_test(label, n_channels, freq, time, d_model, device): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # --- MHR --- - _run_test("MHR (8ch)", n_channels=8, freq=513, time=977, d_model=64, device=device) + _run_test("MHR (8ch)", n_channels=8, freq=513, time=977, d_model=32, device=device) # --- CO2 --- - _run_test("CO2 (4ch)", n_channels=4, freq=513, time=977, d_model=64, device=device) + _run_test("CO2 (4ch)", n_channels=4, freq=513, time=977, d_model=32, device=device) # --- ECE --- - _run_test("ECE (48ch)", n_channels=48, freq=513, time=977, d_model=64, device=device) + _run_test("ECE (48ch)", n_channels=48, freq=513, time=977, d_model=32, device=device) diff --git a/src/tokamak_foundation_model/models/modality/spectrogram_cae1d.py b/src/tokamak_foundation_model/models/modality/spectrogram_cae1d.py new file mode 100644 index 0000000..cd872a1 --- /dev/null +++ b/src/tokamak_foundation_model/models/modality/spectrogram_cae1d.py @@ -0,0 +1,234 @@ +import math +import torch.nn.functional as f + +from torch import nn + + +def cae1d_cr4(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=8) + + +def cae1d_cr8(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=4) + + +def cae1d_cr16(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=2) + + +def cae1d_cr32(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=1) + +def cae1d_cr114(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=32/134) + +def cae1d_cr124(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=64/134) + +def cae1d_cr134(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=100/134) + +def cae1d_cr144(src_channels=103): + return ModifiedConvolutionalAutoencoder1D(src_channels=src_channels, target_bpppc=81/134) + + +class ModifiedConvolutionalAutoencoder1D(nn.Module): + """ + Comment: + Modified version of the below paper to target multiple bitrates. + Title: + 1D-CONVOLUTIONAL AUTOENCODER BASED HYPERSPECTRAL DATA COMPRESSION + Authors: + Kuester, Jannick and Gross, Wolfgang and Middelmann, Wolfgang + Paper: + https://doi.org/10.5194/isprs-archives-XLIII-B1-2021-15-2021 + Cite: + @article{kuester20211d, + title={1D-convolutional autoencoder based hyperspectral data compression}, + author={Kuester, Jannick and Gross, Wolfgang and Middelmann, Wolfgang}, + journal={International Archives of Photogrammetry, Remote Sensing and Spatial Information Sciences}, + volume={43}, + pages={15--21}, + year={2021}, + publisher={Copernicus GmbH} + } + """ + + def __init__(self, src_channels=202, target_bpppc=8): + super(ModifiedConvolutionalAutoencoder1D, self).__init__() + + #assert math.log2(32 // target_bpppc) % 1 == 0 + #self.num_blocks = int(math.log2(32 // target_bpppc)) + self.target_bpppc = target_bpppc + self.compression_ratio = 32.0 / target_bpppc + self.num_blocks = max(1, int(round(math.log2(self.compression_ratio)))) + max_possible_blocks = int(math.log2(src_channels)) + self.num_blocks = min(self.num_blocks, max_possible_blocks) + # Calculate actual achieved compression + self.spectral_downsampling_factor_estimated = 2 ** self.num_blocks + self.actual_bpppc = 32.0 / self.spectral_downsampling_factor_estimated + print(f"Target bpppc: {target_bpppc:.4f}, Actual achieved: {self.actual_bpppc:.4f}") + + self.encoder = nn.Sequential( + nn.Sequential(*[ + nn.Sequential(*[ + nn.Conv1d( + in_channels=1 if i==0 else int(2 ** (self.num_blocks + 5 - i)), + out_channels=int(2 ** (self.num_blocks + 4 - i)), + kernel_size=11, + stride=1, + padding="same", + ), + nn.LeakyReLU(), + nn.MaxPool1d(kernel_size=2), + ]) + for i in range(self.num_blocks) + ]), + nn.Conv1d( + in_channels=32, + out_channels=16, + kernel_size=9, + stride=1, + padding="same", + ), + nn.LeakyReLU(), + nn.Conv1d( + in_channels=16, + out_channels=1, + kernel_size=7, + stride=1, + padding="same", + ), + nn.LeakyReLU(), + ) + + self.decoder = nn.Sequential( + nn.Conv1d( + in_channels=1, + out_channels=16, + kernel_size=7, + stride=1, + padding="same", + ), + nn.LeakyReLU(), + nn.Conv1d( + in_channels=16, + out_channels=32, + kernel_size=9, + stride=1, + padding="same", + ), + nn.LeakyReLU(), + nn.Upsample( + scale_factor=2 + ), + nn.Sequential(*[ + nn.Sequential(*[ + nn.Conv1d( + in_channels=int(2 ** (5 + i)), + out_channels=int(2 ** (6 + i)) if i < self.num_blocks - 1 else 1, + kernel_size=11, + stride=1, + padding="same", + ), + nn.LeakyReLU() if i < self.num_blocks - 1 else nn.Sigmoid(), + nn.Upsample( + scale_factor=2 + ) if i < self.num_blocks - 1 else nn.Identity(), + ]) + for i in range(self.num_blocks) + ]), + ) + + self.src_channels = src_channels + + self.spectral_downsamplings = self.num_blocks + self.spectral_downsampling_factor_estimated = 2 ** self.spectral_downsamplings + + self.spatial_downsamplings = 0 + self.spatial_downsampling_factor = 2 ** self.spatial_downsamplings + + self.latent_channels = int(math.ceil(self.src_channels / 2 ** self.spectral_downsamplings)) + self.spectral_downsampling_factor = self.src_channels / self.latent_channels + self.compression_ratio = self.spectral_downsampling_factor * self.spatial_downsampling_factor ** 2 + self.bpppc = 32.0 / self.compression_ratio + + self.padding_amount = 0 if self.src_channels % self.spectral_downsampling_factor_estimated == 0 \ + else self.spectral_downsampling_factor_estimated - self.src_channels % self.spectral_downsampling_factor_estimated + + def forward(self, x): + n, c, h, w = x.shape + + x = x.permute(0, 2, 3, 1).reshape(-1, c) + if self.padding_amount > 0: + x = f.pad(x, (self.padding_amount, 0)) + x = x.unsqueeze(1) + + y = self.encoder(x) + x_hat = self.decoder(y) + + if self.padding_amount > 0: + x_hat = x_hat[:, :, self.padding_amount:] + x_hat = x_hat.squeeze(1) + x_hat = x_hat.reshape(n, h, w, c).permute(0, 3, 1, 2) + + return x_hat + + def compress(self, x): + n, c, h, w = x.shape + + x = x.permute(0, 2, 3, 1).reshape(-1, c) + if self.padding_amount > 0: + x = f.pad(x, (self.padding_amount, 0)) + x = x.unsqueeze(1) + + y = self.encoder(x) + y = y.squeeze(1) + y = y.reshape(n, h, w, -1).permute(0, 3, 1, 2) + + return y + + def decompress(self, y): + n, c, h, w = y.shape + + y = y.permute(0, 2, 3, 1).reshape(-1, c) + y = y.unsqueeze(1) + x_hat = self.decoder(y) + + if self.padding_amount > 0: + x_hat = x_hat[:, :, self.padding_amount:] + x_hat = x_hat.squeeze(1) + x_hat = x_hat.reshape(n, h, w, -1).permute(0, 3, 1, 2) + + return x_hat + + @classmethod + def from_state_dict(cls, state_dict): + net = cls() + net.load_state_dict(state_dict) + return net + + +if __name__ == '__main__': + # python -m src.tokamak_foundation_model.models.modality.spectrogram_cae1d + import torch + from torchinfo import summary + + model = ModifiedConvolutionalAutoencoder1D() + print(model) + + summary(model, input_size=(2, 202, 128, 128), device='cpu') + + in_tensor = torch.randn(1, 202, 128, 128) + print("in shape:\t\t", in_tensor.shape) + + latent_tensor = model.compress(in_tensor) + print("latent shape:\t\t", latent_tensor.shape) + + out_tensor = model(in_tensor) + print("out shape:\t\t", out_tensor.shape) + + print("in shape = out shape:\t", out_tensor.shape == in_tensor.shape) + + print("real bpppc:\t\t", 32 * torch.numel(latent_tensor) / torch.numel(in_tensor)) + print("model parameter bpppc:\t", model.bpppc) \ No newline at end of file diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index 37fe113..3e993df 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -220,11 +220,20 @@ def train( self.lr_scheduler.step() - self.lr_scheduler.step() - # Logging if self.log_interval is not None: if epoch % self.log_interval == 0: self._log_epoch(epoch, train_loss, val_loss) logger.info("Training complete.") + + def load_checkpoint(self, checkpoint_path=None): + """ + TODO: Modify this as we have more information stored in the checkpoint now. + """ + path = checkpoint_path if checkpoint_path else self.checkpoint_path + if os.path.exists(path): + self.model.load_state_dict(torch.load(path, map_location=self.device)) + print(f"Model loaded from checkpoint: {path}") + else: + print(f"No checkpoint found at: {path}") \ No newline at end of file diff --git a/src/tokamak_foundation_model/utils/drawing.py b/src/tokamak_foundation_model/utils/drawing.py index e549706..0da7514 100644 --- a/src/tokamak_foundation_model/utils/drawing.py +++ b/src/tokamak_foundation_model/utils/drawing.py @@ -61,9 +61,9 @@ def __call__(self, model: torch.nn.Module, epoch: int, train_loss: float, val_lo title = f"Epoch {epoch+1} | Train L1={train_loss:.4f} Val L1={val_loss:.4f}" path = self.drawing_path / f"epoch_{epoch+1:03d}_sample_{i}.png" - # Pick first channel for visualization - inp_vis = inp[0] - out_vis = output[0] + # Visualize the channel in the middle of the signal (usually more activity) + inp_vis = inp[self.half_channel] + out_vis = output[self.half_channel] match self.ndim: case 2: # (C, T) — 1D signals From 5482428943e87788105847b7382d286e7a388f1a Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Tue, 17 Feb 2026 09:50:30 -0500 Subject: [PATCH 15/30] Moved some remaining scripts to the correct subdirectories. --- .../standardize_dataset.py | 2 +- scripts/fast_time_series_reconstruction.py | 190 ----------------- scripts/profile_reconstruction.py | 194 ------------------ .../spectrogram_reconstruction.py | 0 ...train_multimodal_latent_space_predictor.py | 0 5 files changed, 1 insertion(+), 385 deletions(-) rename scripts/{ => data_preparation}/standardize_dataset.py (90%) delete mode 100644 scripts/fast_time_series_reconstruction.py delete mode 100644 scripts/profile_reconstruction.py rename scripts/{ => training}/spectrogram_reconstruction.py (100%) rename scripts/{ => training}/train_multimodal_latent_space_predictor.py (100%) diff --git a/scripts/standardize_dataset.py b/scripts/data_preparation/standardize_dataset.py similarity index 90% rename from scripts/standardize_dataset.py rename to scripts/data_preparation/standardize_dataset.py index cc8f1fe..5f37a48 100644 --- a/scripts/standardize_dataset.py +++ b/scripts/data_preparation/standardize_dataset.py @@ -21,4 +21,4 @@ input_signals=all_input_signals, target_signals=all_input_signals, ) for f in hdf5_files] -stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') +stats = compute_preprocessing_stats(datasets, '../preprocessing_stats.pt') diff --git a/scripts/fast_time_series_reconstruction.py b/scripts/fast_time_series_reconstruction.py deleted file mode 100644 index 808037d..0000000 --- a/scripts/fast_time_series_reconstruction.py +++ /dev/null @@ -1,190 +0,0 @@ -from pathlib import Path -import argparse -import logging - -import torch -import torch.nn as nn -import torch.optim as optim -from torch.utils.data import ConcatDataset, DataLoader - -from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn -from tokamak_foundation_model.data.utils import worker_init_fn -from tokamak_foundation_model.trainer.trainer import UnimodalTrainer -from tokamak_foundation_model.models.model_factory import ( - build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) - -from tokamak_foundation_model.utils import DefaultDrawer - - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def main(): - - ### Settings ### - parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") - parser.add_argument( - "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - default="d_alpha", - help="Signal name to train on" - ) - parser.add_argument( - "--n_fft", type=int, default=1024, help="FFT size", - ) - parser.add_argument( - "--hop_length", type=int, default=256, help="Hop length for STFT.", - ) - parser.add_argument( - "--model", choices=list(MODEL_REGISTRY.keys()), default="fast_time_series", - help="Model type (default: auto-selected from signal)" - ) - parser.add_argument( - "--data_dir", type=str, - default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", - help="Path to HDF5 data directory" - ) - parser.add_argument( - "--stats_path", type=str, - default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", - help="Path to preprocessing stats file" - ) - parser.add_argument( - "--d_model", type=int, default=512, help="Model dimension" - ) - parser.add_argument( - "--n_tokens", type=int, default=140, - help="Number of latent tokens (default: use model default)" - ) - parser.add_argument( - "--batch_size", type=int, default=2, - help="Batch size (for spectrograms, each sample's C channels are processed " - "independently, so effective batch = batch_size * C)" - ) - parser.add_argument( - "--num_workers", type=int, default=4, help="Number of data loader workers" - ) - parser.add_argument( - "--epochs", type=int, default=50, help="Number of training epochs" - ) - parser.add_argument( - "--lr", type=float, default=5e-3, help="Learning rate" - ) - parser.add_argument( - "--weight_decay", type=float, default=0.05, help="AdamW weight decay" - ) - parser.add_argument( - "--warmup_epochs", type=int, default=5, - help="LR warmup epochs (0 to disable scheduler)" - ) - parser.add_argument( - "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" - ) - parser.add_argument( - "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" - ) - parser.add_argument( - "--num_plots", type=int, default=4, - help="Number of reconstruction plots per epoch" - ) - parser.add_argument( - "--log_interval", type=int, default=1, help="Plot every N epochs" - ) - parser.add_argument( - "--resume", action="store_true", default=False, - help="Resume training from checkpoint" - ) - args = parser.parse_args() - - ### Paths ### - signal_name = args.signal - model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] - data_dir = Path(args.data_dir) - statistics_path = Path(args.stats_path) - checkpoint_path = ( - Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" - ) - checkpoint_path.parent.mkdir(parents=True, exist_ok=True) - - logger.info(f"Signal: {signal_name}, Model: {model_name}") - - ### Dataset Setup ### - hdf5_files = sorted(data_dir.glob("*_processed.h5")) - stats = torch.load(statistics_path) - - datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=[signal_name], - target_signals=[signal_name], - n_fft=args.n_fft, - hop_length=args.hop_length, - prediction_mode=False, - ) - for f in hdf5_files - ] - - concatenated_dataset = ConcatDataset(datasets_processed) - - # Not sure if this is elegant - sample_data = next(iter(concatenated_dataset))[signal_name] - n_channels = sample_data.shape[0] - logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") - - ### Model Setup ### - model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, - n_channels=n_channels, kernel_size=3).to(device) - - n_params = sum(p.numel() for p in model.parameters()) - logger.info(f"Model parameters: {n_params:,}") - - optimizer = optim.AdamW( - model.parameters(), - lr=args.lr, - ) - - lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( - optimizer, - T_max=args.epochs, - eta_min=args.min_lr - ) - - loss_fn = nn.L1Loss() - - dataloader = DataLoader( - concatenated_dataset, - batch_size=args.batch_size, - collate_fn=collate_fn, - worker_init_fn=worker_init_fn, - num_workers=args.num_workers, - persistent_workers=args.num_workers > 0, - pin_memory=True, - shuffle=True, - ) - - ### Training ### - drawer = DefaultDrawer(num_plots=args.num_plots) - trainer = UnimodalTrainer( - epochs=args.epochs, - checkpoint_path=checkpoint_path, - model=model, - optimizer=optimizer, - lr_scheduler=lr_scheduler, - loss_fn=loss_fn, - device=device, - drawer=drawer, - log_interval=args.log_interval, - ) - - if args.resume and checkpoint_path.exists(): - logger.info(f"Resuming training from checkpoint: {checkpoint_path}") - trainer.load_checkpoint(checkpoint_path=checkpoint_path) - - trainer.train(dataloader, modality_key=signal_name) - - -if __name__ == "__main__": - main() diff --git a/scripts/profile_reconstruction.py b/scripts/profile_reconstruction.py deleted file mode 100644 index 91500d9..0000000 --- a/scripts/profile_reconstruction.py +++ /dev/null @@ -1,194 +0,0 @@ -from pathlib import Path -import argparse -import logging - -import torch -import torch.nn as nn -import torch.optim as optim -from torch.utils.data import ConcatDataset, DataLoader - -from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn -from tokamak_foundation_model.data.utils import worker_init_fn -from tokamak_foundation_model.trainer.trainer import UnimodalTrainer -from tokamak_foundation_model.models.model_factory import ( - build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) - -from tokamak_foundation_model.utils import DefaultDrawer - - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -def main(): - - ### Settings ### - parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") - parser.add_argument( - "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - default="mse", - help="Signal name to train on" - ) - parser.add_argument( - "--n_fft", type=int, default=1024, help="FFT size", - ) - parser.add_argument( - "--hop_length", type=int, default=256, help="Hop length for STFT.", - ) - parser.add_argument( - "--model", choices=list(MODEL_REGISTRY.keys()), default="profile", - help="Model type (default: auto-selected from signal)" - ) - parser.add_argument( - "--data_dir", type=str, - default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", - help="Path to HDF5 data directory" - ) - parser.add_argument( - "--stats_path", type=str, - default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", - help="Path to preprocessing stats file" - ) - parser.add_argument( - "--d_model", type=int, default=512, help="Model dimension" - ) - parser.add_argument( - "--n_tokens", type=int, default=140, - help="Number of latent tokens (default: use model default)" - ) - parser.add_argument( - "--batch_size", type=int, default=2, - help="Batch size (for spectrograms, each sample's C channels are processed " - "independently, so effective batch = batch_size * C)" - ) - parser.add_argument( - "--num_workers", type=int, default=4, help="Number of data loader workers" - ) - parser.add_argument( - "--epochs", type=int, default=50, help="Number of training epochs" - ) - parser.add_argument( - "--lr", type=float, default=5e-3, help="Learning rate" - ) - parser.add_argument( - "--weight_decay", type=float, default=0.01, help="AdamW weight decay" - ) - parser.add_argument( - "--warmup_epochs", type=int, default=5, - help="LR warmup epochs (0 to disable scheduler)" - ) - parser.add_argument( - "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" - ) - parser.add_argument( - "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" - ) - parser.add_argument( - "--num_plots", type=int, default=4, - help="Number of reconstruction plots per epoch" - ) - parser.add_argument( - "--log_interval", type=int, default=1, help="Plot every N epochs" - ) - parser.add_argument( - "--resume", action="store_true", default=False, - help="Resume training from checkpoint" - ) - args = parser.parse_args() - - ### Paths ### - signal_name = args.signal - model_name = args.model or SIGNAL_MODEL_DEFAULTS[signal_name] - data_dir = Path(args.data_dir) - statistics_path = Path(args.stats_path) - checkpoint_path = ( - Path(args.checkpoint_dir) / f"{signal_name}_{model_name}" / "checkpoint.pth" - ) - checkpoint_path.parent.mkdir(parents=True, exist_ok=True) - - logger.info(f"Signal: {signal_name}, Model: {model_name}") - - ### Dataset Setup ### - hdf5_files = sorted(data_dir.glob("*_processed.h5")) - stats = torch.load(statistics_path) - - datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=[signal_name], - target_signals=[signal_name], - n_fft=args.n_fft, - hop_length=args.hop_length, - prediction_mode=False, - ) - for f in hdf5_files - ] - - concatenated_dataset = ConcatDataset(datasets_processed) - - # Not sure if this is elegant - sample_data = next(iter(concatenated_dataset))[signal_name] - logger.info(f"Sample data shape: {sample_data.shape}") - n_spatial_points = sample_data.shape[0] - n_time_points = sample_data.shape[1] - logger.info(f"n_spatial_points: {n_spatial_points}, n_time_points: {n_time_points}") - ### Model Setup ### - model = build_model(model_name, d_model=args.d_model, n_tokens=args.n_tokens, - n_channels=1, n_spatial_points=n_spatial_points, - n_time_points=n_time_points, kernel_size=3) - - model = model.to(device) - - n_params = sum(p.numel() for p in model.parameters()) - logger.info(f"Model parameters: {n_params:,}") - - optimizer = optim.AdamW( - model.parameters(), - lr=args.lr, - ) - - lr_scheduler = optim.lr_scheduler.CosineAnnealingLR( - optimizer, - T_max=args.epochs, - eta_min=args.min_lr - ) - - loss_fn = nn.L1Loss() - - dataloader = DataLoader( - concatenated_dataset, - batch_size=args.batch_size, - collate_fn=collate_fn, - worker_init_fn=worker_init_fn, - num_workers=args.num_workers, - persistent_workers=args.num_workers > 0, - pin_memory=True, - shuffle=True, - ) - - ### Training ### - drawer = DefaultDrawer(num_plots=args.num_plots) - trainer = UnimodalTrainer( - epochs=args.epochs, - checkpoint_path=checkpoint_path, - model=model, - optimizer=optimizer, - lr_scheduler=lr_scheduler, - loss_fn=loss_fn, - device=device, - drawer=drawer, - log_interval=args.log_interval, - ) - - if args.resume and checkpoint_path.exists(): - logger.info(f"Resuming training from checkpoint: {checkpoint_path}") - trainer.load_checkpoint(checkpoint_path=checkpoint_path) - - trainer.train(dataloader, modality_key=signal_name) - - -if __name__ == "__main__": - main() diff --git a/scripts/spectrogram_reconstruction.py b/scripts/training/spectrogram_reconstruction.py similarity index 100% rename from scripts/spectrogram_reconstruction.py rename to scripts/training/spectrogram_reconstruction.py diff --git a/scripts/train_multimodal_latent_space_predictor.py b/scripts/training/train_multimodal_latent_space_predictor.py similarity index 100% rename from scripts/train_multimodal_latent_space_predictor.py rename to scripts/training/train_multimodal_latent_space_predictor.py From abb30578292ff3f253417a2854fda1e18fd90579 Mon Sep 17 00:00:00 2001 From: Peter Steiner <61472983+renierts@users.noreply.github.com> Date: Tue, 17 Feb 2026 16:37:21 -0500 Subject: [PATCH 16/30] Still working on preparing the dataset. This is not ready to push. Preparation to moving to Stellar. --- pixi.lock | 158 +++++++--- pyproject.toml | 2 + .../data_preparation/make_processing_stats.py | 18 +- .../data/data_loader.py | 274 ++++++++++++++---- 4 files changed, 351 insertions(+), 101 deletions(-) diff --git a/pixi.lock b/pixi.lock index 3973dfb..53a9c4a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -12,8 +12,10 @@ environments: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 + - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_101.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda @@ -29,11 +31,17 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.1-h35e630c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.14-hd63d673_3_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py311h3778330_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl @@ -98,7 +106,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b5/09/6ea3ea725f82e1e76684f0708bbedd871fc96da89945adeba65c3835a64c/nvidia_nvshmem_cu12-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/62/fb/89319812eb1d714bfc04b7f177895caeba8ab4a37ef6712db75ed786e2e0/pandas-3.0.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl @@ -112,7 +119,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b1/c4/2a6fe5111a01005fc7af3878259ce17684fabb8852815eda6225620f3c59/pyzmq-27.1.0-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a4/3c/87ca0a02736d16b6262921425e84b48984e77d8e4e572c9072ce96e66c30/regex-2026.1.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl @@ -134,14 +140,15 @@ environments: - pypi: https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.2-h38cb7af_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-hcf2aa1b_0.conda @@ -149,11 +156,17 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.2-h1ae2325_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.1-hd24854e_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.14-h18782d2_3_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py311hc290fe0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl @@ -201,7 +214,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/95/d64f680ea1fc56d165457287e0851d6708800f9fcea346fc1b9957942ee6/numexpr-2.14.1-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/dd/5e/e04a547ad0f0183bf151fd7c7a477468e3b85ff2ad231c566389e6cc9587/pandas-3.0.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl @@ -215,7 +227,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/06/5d/305323ba86b284e6fcb0d842d6adaa2999035f70f8c38a9b6d21ad28c3d4/pyzmq-27.1.0-cp311-cp311-macosx_10_15_universal2.whl - pypi: https://files.pythonhosted.org/packages/a0/f9/5f1fd077d106ca5655a0f9ff8f25a1ab55b92128b5713a91ed7134ff688e/regex-2026.1.15-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl @@ -236,27 +247,34 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/66/57042d4b0f1ede8046d7ae6409bf3640df996e9cbc3fe20467aa29badc54/transformers-5.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ win-64: + - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h3d046cb_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.2-hfd05255_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.2-hf5d6505_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.1-hf411b9b_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_3_cpython.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py311h3f79411_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h6ed50ae_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/ucrt-10.0.26100.0-h57928b3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc-14.3-h41ae7f8_34.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_34.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_34.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl @@ -304,7 +322,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/72/4ca9bd97b2eb6dce9f5e70a3b6acec1a93e1fb9b079cb4cba2cdfbbf295d/numexpr-2.14.1-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl - - pypi: https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/51/27/bf9436dd0a4fc3130acec0828951c7ef96a0631969613a9a35744baf27f6/pandas-3.0.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl @@ -316,7 +333,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/23/6d/d8d92a0eb270a925c9b4dd039c0b4dc10abc2fcbc48331788824ef113935/pyzmq-27.1.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/0f/19/772cf8b5fc803f5c89ba85d8b1870a1ca580dc482aa030383a9289c82e44/regex-2026.1.15-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl @@ -337,7 +353,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/66/57042d4b0f1ede8046d7ae6409bf3640df996e9cbc3fe20467aa29badc54/transformers-5.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl @@ -358,6 +373,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/aiohappyeyeballs-2.6.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/aiohttp-3.13.3-py311h55b9665_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/aiosignal-1.4.0-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.12.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argon2-cffi-25.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/argon2-cffi-bindings-25.1.0-py311h49ec1c0_2.conda @@ -420,6 +436,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/httpcore-1.0.9-pyh29332c3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/httpx-0.28.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-75.1-he02047a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda @@ -519,6 +536,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/notebook-shim-0.2.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/numcodecs-0.16.5-py311hed34c8f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py311h64a7726_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.31-pthreads_h6ec200e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.1-h35e630c_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/orc-2.2.1-hd747db4_0.conda @@ -733,6 +751,17 @@ packages: version: 0.0.4 sha256: 571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 requires_python: '>=3.8' +- conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 + sha256: b91f8ab4ac2b48972fbee1fc8e092cc452fdf59156e4ff2322c94bbf73650f94 + md5: c88eaec8de9ae1fa161205aa18e7a5b1 + depends: + - python >=3.6 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/antlr4-python3-runtime?source=hash-mapping + size: 101065 + timestamp: 1638309284042 - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl name: anyio version: 4.12.1 @@ -1737,7 +1766,7 @@ packages: - pypi: ./ name: faith version: 26.1.dev0 - sha256: 82a4b0f6c17173c3bda7c4c7cf9f0ba6fef9e8e8207586a93c26925a8ce5246b + sha256: b8c8cb7c861aef475e478a2e13862ae2f0af650b35644f910f46fed6e8b2cf3f requires_dist: - einops>=0.8.2,<0.9 - h5py>=3.15.1,<4 @@ -2332,6 +2361,20 @@ packages: - types-tqdm ; extra == 'dev' - types-urllib3 ; extra == 'dev' requires_python: '>=3.9.0' +- conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda + sha256: 40b4469bd65e0156de1136ae8b265f5d2d72f14b8d431e009836d59438339ee8 + md5: a189dd36bcaaf4c7647deb2dcb4e1b05 + depends: + - antlr-python-runtime 4.9.* + - omegaconf >=2.2,<2.4 + - packaging + - python >=3.9 + license: MIT + license_family: MIT + purls: + - pkg:pypi/hydra-core?source=hash-mapping + size: 110015 + timestamp: 1736934833060 - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda sha256: 77af6f5fe8b62ca07d09ac60127a30d9069fdc3c68d6b256754d0ffb1f7779f8 md5: 8e6923fc12f1fe8f8c4e5c9f343256ac @@ -4549,6 +4592,20 @@ packages: version: 12.8.90 sha256: 5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f requires_python: '>=3' +- conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda + sha256: df806841be847e5287b22b6ae7f380874f81ea51f1b51ae14a570f3385c7b133 + md5: 23cc056834cab53849b91f78d6ee3ea0 + depends: + - antlr-python-runtime 4.9.* + - python >=3.7 + - pyyaml >=5.1.0 + - typing_extensions + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/omegaconf?source=hash-mapping + size: 166453 + timestamp: 1670575519562 - conda: https://conda.anaconda.org/conda-forge/linux-64/openblas-0.3.31-pthreads_h6ec200e_0.conda sha256: 030219c939832ffc6092ca2a83f2182ee26adf66c0089c9bceb34484eeb887a0 md5: 5d4794b11a5af3c1e7f990026d08a9cf @@ -4625,11 +4682,6 @@ packages: - pkg:pypi/overrides?source=hash-mapping size: 30139 timestamp: 1734587755455 -- pypi: https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl - name: packaging - version: '26.0' - sha256: b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529 - requires_python: '>=3.8' - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda sha256: c1fc0f953048f743385d31c468b4a678b3ad20caffdeaa94bed85ba63049fd58 md5: b76541e68fea4d511b1ac46a28dcd2c6 @@ -5718,21 +5770,6 @@ packages: - pkg:pypi/pytz?source=hash-mapping size: 189015 timestamp: 1742920947249 -- pypi: https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl - name: pyyaml - version: 6.0.3 - sha256: 652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 - requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - name: pyyaml - version: 6.0.3 - sha256: b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d - requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl - name: pyyaml - version: 6.0.3 - sha256: 9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf - requires_python: '>=3.8' - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py311h3778330_1.conda sha256: c9a6cd2c290d7c3d2b30ea34a0ccda30f770e8ddb2937871f2c404faf60d0050 md5: a24add9a3bababee946f3bc1c829acfe @@ -5748,6 +5785,37 @@ packages: - pkg:pypi/pyyaml?source=compressed-mapping size: 206190 timestamp: 1770223702917 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py311hc290fe0_1.conda + sha256: 984e73d7957460689e10533059de8adb38a308853d298900a37acc58edd84cec + md5: e4b908da7cd496b3fa6798c0f60a2a19 + depends: + - __osx >=11.0 + - python >=3.11,<3.12.0a0 + - python >=3.11,<3.12.0a0 *_cpython + - python_abi 3.11.* *_cp311 + - yaml >=0.2.5,<0.3.0a0 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyyaml?source=compressed-mapping + size: 192948 + timestamp: 1770223655988 +- conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py311h3f79411_1.conda + sha256: 301c3ba100d25cd5ae37895988ee3ab986210d4d972aa58efed948fbe857773d + md5: a0153c033dc55203e11d1cac8f6a9cf2 + depends: + - python >=3.11,<3.12.0a0 + - python_abi 3.11.* *_cp311 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - yaml >=0.2.5,<0.3.0a0 + license: MIT + license_family: MIT + purls: + - pkg:pypi/pyyaml?source=compressed-mapping + size: 187108 + timestamp: 1770223467913 - pypi: https://files.pythonhosted.org/packages/06/5d/305323ba86b284e6fcb0d842d6adaa2999035f70f8c38a9b6d21ad28c3d4/pyzmq-27.1.0-cp311-cp311-macosx_10_15_universal2.whl name: pyzmq version: 27.1.0 @@ -6935,11 +7003,6 @@ packages: requires_dist: - typer>=0.22.0 requires_python: '>=3.9' -- pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl - name: typing-extensions - version: 4.15.0 - sha256: f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 - requires_python: '>=3.9' - conda: https://conda.anaconda.org/conda-forge/noarch/typing-extensions-4.15.0-h396c80c_0.conda sha256: 7c2df5721c742c2a47b2c8f960e718c930031663ac1174da67c1ed5999f7938c md5: edd329d7d3a4ab45dcf905899a7a6115 @@ -7241,6 +7304,31 @@ packages: purls: [] size: 85189 timestamp: 1753484064210 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda + sha256: b03433b13d89f5567e828ea9f1a7d5c5d697bf374c28a4168d71e9464f5dafac + md5: 78a0fe9e9c50d2c381e8ee47e3ea437d + depends: + - __osx >=11.0 + license: MIT + license_family: MIT + purls: [] + size: 83386 + timestamp: 1753484079473 +- conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda + sha256: 80ee68c1e7683a35295232ea79bcc87279d31ffeda04a1665efdb43cbd50a309 + md5: 433699cba6602098ae8957a323da2664 + depends: + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + license: MIT + license_family: MIT + purls: [] + size: 63944 + timestamp: 1753484092156 - conda: https://conda.anaconda.org/conda-forge/linux-64/yarl-1.22.0-py311h3778330_0.conda sha256: 6cddfbe838aab2d374a22f0c202f473a1d81c43e8fda25c5aa18fdcbc4f61679 md5: c8213cef4057bc5a733d68d36e9b6366 diff --git a/pyproject.toml b/pyproject.toml index a489432..adb445b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,8 @@ torchvision = { version = ">=0.20.1", index = "https://download.pytorch.org/whl/ [tool.pixi.dependencies] python = ">=3.11,<3.12" +omegaconf = ">=2.3.0,<3" +hydra-core = ">=1.3.2,<2" [tool.pixi.feature.fdp] platforms = ["linux-64"] diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index 53bc61f..a6ddfa9 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -3,18 +3,16 @@ TokamakH5Dataset, compute_preprocessing_stats) def main(): - # hdf5_files = sorted( - # Path( - # "/scratch/gpfs/EKOLEMEN/foundation_model" - # ).glob("*_processed.h5") - # ) - hdf5_files = sorted( - Path( - "/scratch/gpfs/EKOLEMEN/foundation_model" - ).glob("*_processed.h5") + Path( + "C:/Users/admin/PycharmProjects/FusionAIHub/scripts/training/" + ).glob("*_processed.h5") ) + # hdf5_files = sorted( + # Path("/scratch/gpfs/EKOLEMEN/foundation_model").glob("*_processed.h5") + # ) + all_input_signals = [ "mhr", "ece", "co2", "bes", # spectrograms "gas", "ech", "pin", "tin", # actuators @@ -34,4 +32,4 @@ def main(): if __name__ == "__main__": # python scripts/data_preparation/make_processing_stats.py - main() \ No newline at end of file + main() diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index 10045b2..c5428fe 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -69,9 +69,8 @@ def compute_preprocessing_stats( # Collect values values = [] - indices = torch.randperm(len(combined))[:num_samples] - for idx in tqdm(indices): + for idx in tqdm(range(len(combined))): batch = combined[int(idx)] if config.name in batch['inputs']: values.append(batch['inputs'][config.name]) @@ -301,7 +300,7 @@ def __init__( self.h5_file = None with h5py.File(self.hdf5_path, "r") as f: - self.duration = self._compute_duration_from_handle(f) + self.duration, self.t0_indices = self._compute_duration_and_t0_indices(f) # In prediction mode, reduce length to ensure extended window fits if self.prediction_mode: @@ -314,6 +313,138 @@ def __init__( self.n_freq_bins = n_fft // 2 + 1 self.stft_window = torch.hann_window(n_fft) + def _find_t0_index(self, xdata_ms: np.ndarray) -> tuple[int, float]: + """ + Find the index and exact time of t=0 in xdata. + + Parameters + ---------- + xdata_ms : np.ndarray + Array of timestamps in milliseconds + + Returns + ------- + tuple[int, float] + (index, actual_time_ms) where: + - index: Index closest to t=0, or -1 if all data is before t=0 + - actual_time_ms: The actual timestamp at that index + """ + if len(xdata_ms) == 0: + return -1, 0.0 + + if len(xdata_ms) == 1: + # Single sample - use it if >= 0, else -1 + if xdata_ms[0] >= 0: + return 0, xdata_ms[0] + else: + return -1, xdata_ms[0] + + # All data before t=0 + if xdata_ms[-1] < 0: + return -1, xdata_ms[-1] + + # All data after t=0 (first sample is already past t=0) + if xdata_ms[0] > 0: + return 0, xdata_ms[0] + + # t=0 is within range - find nearest index using binary search + idx = np.searchsorted(xdata_ms, 0) + + # searchsorted returns insertion point + # Check if previous index is closer to 0 + if idx > 0 and idx < len(xdata_ms): + if abs(xdata_ms[idx - 1]) < abs(xdata_ms[idx]): + idx = idx - 1 + elif idx >= len(xdata_ms): + idx = len(xdata_ms) - 1 + + return idx, xdata_ms[idx] + + def _compute_duration_and_t0_indices(self, f: h5py.File) -> tuple[float, dict]: + """ + Compute duration from t=0 and store info about where t=0 occurs for each signal. + + Returns + ------- + tuple[float, dict] + (max_duration_from_t0, {signal_name: {'index': int, 'time_s': float}}) + where: + - 'index': first index where xdata >= 0 + - 'time_s': actual time value (in seconds) at that index + """ + max_duration = 0.0 + t0_indices = {} + + # Process signals + for config in self.signal_configs: + for key_path in config.hdf5_keys: + try: + parts = key_path.split("/") + curr = f + for part in parts: + curr = curr[part] + + xdata_ms = curr["xdata"][:] + + if len(xdata_ms) < 2: + continue + + # Find first index where t >= 0 + t0_idx = np.searchsorted(xdata_ms, 0, side="left") + + # If all data is before t=0, skip + if t0_idx >= len(xdata_ms): + continue + + # Store both index and actual time at that index + t0_indices[config.name] = { + "index": int(t0_idx), + "time_s": float(xdata_ms[t0_idx]) / 1000.0, + } + + # Duration from t=0 to end + duration_s = (xdata_ms[-1] - 0.0) / 1000.0 + max_duration = max(max_duration, duration_s) + + break + + except (KeyError, ValueError): + continue + + # Process movies + for movie_config in self.movie_configs: + for key_path in movie_config.hdf5_keys: + try: + parts = key_path.split("/") + curr = f + for part in parts: + curr = curr[part] + + xdata_ms = curr["xdata"][:] + + if len(xdata_ms) < 2: + continue + + t0_idx = np.searchsorted(xdata_ms, 0, side="left") + + if t0_idx >= len(xdata_ms): + continue + + t0_indices[movie_config.name] = { + "index": int(t0_idx), + "time_s": float(xdata_ms[t0_idx]) / 1000.0, + } + + duration_s = (xdata_ms[-1] - 0.0) / 1000.0 + max_duration = max(max_duration, duration_s) + + break + + except (KeyError, ValueError): + continue + + return max(max_duration, 1.0), t0_indices + def _update_preprocessing_stats(self): """Update preprocessing configs with loaded statistics.""" for config in self.signal_configs: @@ -410,24 +541,6 @@ def _apply_preprocessing( return tensor - def _compute_duration_from_handle(self, f: h5py.File) -> float: - """Compute total duration from an open HDF5 file handle.""" - try: - for key_path in ["mhr/xdata", "ece/xdata", "co2/xdata"]: - try: - parts = key_path.split("/") - data = f - for part in parts: - data = data[part] - xdata = data[:] - return (xdata[-1] - xdata[0]) / 1000.0 - except (KeyError, ValueError): - continue - except Exception as e: - print(f"Warning: Could not determine duration from {self.hdf5_path}: {e}") - - return 1.0 # Default fallback - def _open_hdf5(self): """Open HDF5 file for this worker with optimized cache settings.""" if self.h5_file is None: @@ -441,12 +554,38 @@ def _open_hdf5(self): def _load_signal_raw( self, f: h5py.File, config: SignalConfig, t_start: float, t_end: float ) -> torch.Tensor: - """Load raw signal at native sampling rate within time window. - - Returns: - Array of shape (time, channels) at native sampling rate """ - # Try to find the signal in HDF5 + Load raw signal at native sampling rate within time window. + + Parameters + ---------- + f : h5py.File + Open HDF5 file handle + config : SignalConfig + Signal configuration + t_start : float + Start time in seconds (relative to t=0) + t_end : float + End time in seconds (relative to t=0) + + Returns + ------- + torch.Tensor + Array of shape (time_samples, channels) at native sampling rate + """ + duration_s = t_end - t_start + + # Step 1: Check if signal has data after t=0 + if config.name not in self.t0_indices: + return torch.zeros( + (round(duration_s * config.target_fs), config.num_channels) + ) + + t0_info = self.t0_indices[config.name] + t0_idx = t0_info["index"] + t0_time_s = t0_info["time_s"] + + # Step 2: Find the signal in HDF5 data_group = None for key_path in config.hdf5_keys: try: @@ -459,52 +598,75 @@ def _load_signal_raw( except KeyError: continue - # Extract data with time slicing + if data_group is None: + return torch.zeros( + (round(duration_s * config.target_fs), config.num_channels) + ) + ydata_ds = data_group["ydata"] xdata_ds = data_group["xdata"] - # Load only first and last timestamp - t0 = xdata_ds[0] / 1000.0 - t1 = xdata_ds[-1] / 1000.0 + # Load first and last timestamp to compute sampling rate + t_first = xdata_ds[0] / 1000.0 + t_last = xdata_ds[-1] / 1000.0 n_samples = xdata_ds.shape[0] - fs_raw = (n_samples - 1) / (t1 - t0) - duration_s = t_end - t_start + if n_samples < 2 or t_last == t_first: + return torch.zeros( + (round(duration_s * config.target_fs), config.num_channels) + ) - ydata = np.zeros( - (max(1, round(duration_s * fs_raw)), config.num_channels), dtype=np.float32 - ) + fs_raw = (n_samples - 1) / (t_last - t_first) - start_idx = max(0, int((t_start - t0) * fs_raw)) - end_idx = min(n_samples, int((t_end - t0) * fs_raw)) + # Step 3: Initialize output with zeros at raw sampling rate + output = np.zeros( + (round(duration_s * fs_raw), config.num_channels), dtype=np.float32 + ) - if end_idx > start_idx: - data = ydata_ds[start_idx:end_idx] + # Step 4: Calculate HDF5 indices for requested time range + # xdata[t0_idx] = t0_time_s (actual time, e.g., 0.005s if first sample is at 5ms) + # To find data at user's t_start: + # We want: xdata[i] ≈ t_start + # We know: xdata[i] ≈ t0_time_s + (i - t0_idx) / fs_raw + # Solving: i ≈ t0_idx + (t_start - t0_time_s) * fs_raw + hdf5_start = t0_idx + round((t_start - t0_time_s) * fs_raw) + hdf5_end = t0_idx + round((t_end - t0_time_s) * fs_raw) + + # Clamp to valid HDF5 range + hdf5_start = max(0, min(hdf5_start, n_samples)) + hdf5_end = max(0, min(hdf5_end, n_samples)) + + # Step 5: If there's data to load + if hdf5_start < hdf5_end: + # Load from HDF5 + data = ydata_ds[hdf5_start:hdf5_end] np.nan_to_num(data, copy=False, nan=0.0) - # Compute offset based on actual start time - actual_t_start = t0 + start_idx / fs_raw - idx_1 = round((actual_t_start - t_start) * fs_raw) - idx_2 = idx_1 + data.shape[0] + # Calculate what time range the loaded data represents + # xdata[hdf5_start] ≈ t0_time_s + (hdf5_start - t0_idx) / fs_raw + loaded_t_start = t0_time_s + (hdf5_start - t0_idx) / fs_raw - # Clamp to array bounds + # Position in output (which represents [t_start, t_end]) + output_start = round((loaded_t_start - t_start) * fs_raw) + output_end = output_start + data.shape[0] + + # Clamp to output bounds src_start = 0 src_end = data.shape[0] - if idx_1 < 0: - src_start = -idx_1 - idx_1 = 0 - if idx_2 > ydata.shape[0]: - src_end -= idx_2 - ydata.shape[0] - idx_2 = ydata.shape[0] + if output_start < 0: + src_start = -output_start + output_start = 0 + if output_end > output.shape[0]: + src_end -= output_end - output.shape[0] + output_end = output.shape[0] - if (idx_1 == 0 and idx_2 == ydata.shape[0] - and src_start == 0 and src_end == data.shape[0]): - ydata = data # No copy needed - else: - ydata[idx_1:idx_2] = data[src_start:src_end] + # Copy data to output + if src_start < src_end and output_start < output_end: + output[output_start:output_end] = data[src_start:src_end] - tensor = torch.from_numpy(ydata).float() + # Step 6: Convert to tensor and resample to target frequency + tensor = torch.from_numpy(output).float() tensor = ( F.interpolate( From 86aca347d615a7854ed730072cf57fb84c0aa7de Mon Sep 17 00:00:00 2001 From: renierts Date: Thu, 19 Feb 2026 12:57:48 -0500 Subject: [PATCH 17/30] Updated the data loader. Bugfix for loading the correct slices from H5 files. Implemented calculating incremental statistics. Corrected values in the modality configuration. Removed redundant script standardize_dataset.py --- pixi.lock | 574 +++++++++++++++++- pyproject.toml | 2 + .../data_preparation/make_processing_stats.py | 14 +- .../data_preparation/standardize_dataset.py | 24 - .../data/config/modalities/modalities.yaml | 22 +- .../data/data_loader.py | 470 ++++++++------ .../data/prepare_data.py | 113 +++- .../trainer/trainer.py | 94 +-- 8 files changed, 1008 insertions(+), 305 deletions(-) delete mode 100644 scripts/data_preparation/standardize_dataset.py diff --git a/pixi.lock b/pixi.lock index 53a9c4a..161a9be 100644 --- a/pixi.lock +++ b/pixi.lock @@ -15,22 +15,30 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.20-py311hc665b79_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_101.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-5_h4a7cf45_openblas.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-5_h0358290_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_17.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_17.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_17.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-h68bc16d_17.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_17.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-5_h47877c9_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.30-pthreads_h94d23a6_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.2-hf4e2dac_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_17.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.4.2-py311h2e04523_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.1-h35e630c_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda @@ -38,6 +46,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py311h3778330_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/scipy-1.17.0-py311hbe70eeb_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda @@ -55,7 +64,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0b/02/4dbe7568a42e46582248942f54dc64ad094769532adbe21e525e4edf7bc4/cuda_pathfinder-1.3.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl @@ -90,7 +98,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4c/1a/edbe839109518364ac0bd9e918cf874c755bb2c128040e920f198c494263/numexpr-2.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/1b/46/6fa4ea94f1ddf969b2ee941290cca6f1bfac92b53c76ae5f44afe17ceb69/numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl @@ -145,17 +152,29 @@ environments: - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ osx-arm64: + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/_openmp_mutex-4.5-7_kmp_llvm.conda - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/debugpy-1.8.20-py311h8948835_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.2-h38cb7af_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libblas-3.11.0-5_h51639a9_openblas.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcblas-3.11.0-5_hb0561ab_openblas.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-h55c6f16_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-hcf2aa1b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgcc-15.2.0-hcbb3090_17.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-h07b0088_17.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-hdae7583_17.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.11.0-5_hd9741b5_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.2-h8088a28_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenblas-0.3.30-openmp_ha158390_4.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.2-h1ae2325_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-21.1.8-h4a912ad_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/numpy-2.4.2-py311had1e860_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.1-hd24854e_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda @@ -163,6 +182,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py311hc290fe0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/scipy-1.17.0-py311he9931d0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda @@ -178,7 +198,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl @@ -213,7 +232,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/95/d64f680ea1fc56d165457287e0851d6708800f9fcea346fc1b9957942ee6/numexpr-2.14.1-cp311-cp311-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dd/5e/e04a547ad0f0183bf151fd7c7a477468e3b85ff2ad231c566389e6cc9587/pandas-3.0.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl @@ -255,18 +273,33 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/debugpy-1.8.20-py311h5dfdfe8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/icu-78.2-h637d24d_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libblas-3.11.0-5_hf2e6a31_mkl.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libcblas-3.11.0-5_h2a3cdd5_mkl.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h3d046cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.2-default_h4379cf1_1000.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.11.0-5_hf9ab0e9_mkl.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.2-hfd05255_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.2-hf5d6505_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libwinpthread-12.0.0.r4.gg4f2fc60ca-h57928b3_10.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-16-2.15.1-h3cfd58e_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-2.15.1-h779ef1b_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/llvm-openmp-21.1.8-h4fa8253_0.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/mkl-2025.3.0-hac47afa_455.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/numpy-2.4.2-py311h80b3fa1_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.1-hf411b9b_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_3_cpython.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py311h3f79411_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/scipy-1.17.0-py311h9c22a71_1.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/tbb-2022.3.0-h3155e25_2.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h6ed50ae_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda @@ -286,7 +319,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/d5/92/1cb532e88560cbee973396254b21bece8c5d7c2ece958a67afa08c9f10dc/debugpy-1.8.20-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl @@ -321,7 +353,6 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/72/4ca9bd97b2eb6dce9f5e70a3b6acec1a93e1fb9b079cb4cba2cdfbbf295d/numexpr-2.14.1-cp311-cp311-win_amd64.whl - - pypi: https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/51/27/bf9436dd0a4fc3130acec0828951c7ef96a0631969613a9a35744baf27f6/pandas-3.0.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl @@ -701,6 +732,17 @@ packages: purls: [] size: 23621 timestamp: 1650670423406 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/_openmp_mutex-4.5-7_kmp_llvm.conda + build_number: 7 + sha256: 7acaa2e0782cad032bdaf756b536874346ac1375745fb250e9bdd6a48a7ab3cd + md5: a44032f282e7d2acdeb1c240308052dd + depends: + - llvm-openmp >=9.0.1 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 8325 + timestamp: 1764092507920 - conda: https://conda.anaconda.org/conda-forge/noarch/aiohappyeyeballs-2.6.1-pyhd8ed1ab_0.conda sha256: 7842ddc678e77868ba7b92a726b437575b23aaec293bca0d40826f1026d90e27 md5: 18fd895e0e775622906cdabfc3cf0fb4 @@ -1647,16 +1689,6 @@ packages: - pytest-cov ; extra == 'tests' - pytest-xdist ; extra == 'tests' requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/d5/92/1cb532e88560cbee973396254b21bece8c5d7c2ece958a67afa08c9f10dc/debugpy-1.8.20-cp311-cp311-win_amd64.whl - name: debugpy - version: 1.8.20 - sha256: 1f7650546e0eded1902d0f6af28f787fa1f1dbdbc97ddabaf1cd963a405930cb - requires_python: '>=3.8' -- pypi: https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl - name: debugpy - version: 1.8.20 - sha256: 5be9bed9ae3be00665a06acaa48f8329d2b9632f15fd09f6a9a8c8d9907e54d7 - requires_python: '>=3.8' - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.20-py311hc665b79_0.conda sha256: e69be2be543c4d4898895d8aebe758bc683c5a1198583ad676f5719782a07131 md5: 400e4667a12884216df869cad5fb004b @@ -1672,6 +1704,36 @@ packages: - pkg:pypi/debugpy?source=hash-mapping size: 2733654 timestamp: 1769744984842 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/debugpy-1.8.20-py311h8948835_0.conda + sha256: 093b015e9abf27fb4d3b4f7e52417d35cd69a99fab8b95ec5c6c3983275c46ba + md5: 150c921424bc9f08c0378f8a6ae58d05 + depends: + - python + - __osx >=11.0 + - libcxx >=19 + - python 3.11.* *_cpython + - python_abi 3.11.* *_cp311 + license: MIT + license_family: MIT + purls: + - pkg:pypi/debugpy?source=hash-mapping + size: 2668163 + timestamp: 1769745020016 +- conda: https://conda.anaconda.org/conda-forge/win-64/debugpy-1.8.20-py311h5dfdfe8_0.conda + sha256: 661e5c582b1f853a46a78d4bb6e55f2bfdac66e68d015e111f1580a11c28abbf + md5: 683be2cd10e80a367790b3083ce529b7 + depends: + - python + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - python_abi 3.11.* *_cp311 + license: MIT + license_family: MIT + purls: + - pkg:pypi/debugpy?source=hash-mapping + size: 3940002 + timestamp: 1769745017274 - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl name: decorator version: 5.2.1 @@ -1766,7 +1828,7 @@ packages: - pypi: ./ name: faith version: 26.1.dev0 - sha256: b8c8cb7c861aef475e478a2e13862ae2f0af650b35644f910f46fed6e8b2cf3f + sha256: d143d15dacb53dea0f310e30e110adc36cded0de714eedb798a1145ffea4c3ea requires_dist: - einops>=0.8.2,<0.9 - h5py>=3.15.1,<4 @@ -2420,6 +2482,18 @@ packages: purls: [] size: 12358010 timestamp: 1767970350308 +- conda: https://conda.anaconda.org/conda-forge/win-64/icu-78.2-h637d24d_0.conda + sha256: 5a41fb28971342e293769fc968b3414253a2f8d9e30ed7c31517a15b4887246a + md5: 0ee3bb487600d5e71ab7d28951b2016a + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + purls: [] + size: 13222158 + timestamp: 1767970128854 - pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl name: idna version: '3.11' @@ -3281,6 +3355,24 @@ packages: purls: [] size: 483116 timestamp: 1759482133380 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-5_h4a7cf45_openblas.conda + build_number: 5 + sha256: 18c72545080b86739352482ba14ba2c4815e19e26a7417ca21a95b76ec8da24c + md5: c160954f7418d7b6e87eaf05a8913fa9 + depends: + - libopenblas >=0.3.30,<0.3.31.0a0 + - libopenblas >=0.3.30,<1.0a0 + constrains: + - mkl <2026 + - liblapack 3.11.0 5*_openblas + - libcblas 3.11.0 5*_openblas + - blas 2.305 openblas + - liblapacke 3.11.0 5*_openblas + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 18213 + timestamp: 1765818813880 - conda: https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-7_hc00574d_netlib.conda build_number: 7 sha256: 464608528e7b188fa3a602c503c7f73b3b446bbfd7b259d1c8b56470c34166fc @@ -3300,6 +3392,40 @@ packages: purls: [] size: 222771 timestamp: 1763440535188 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libblas-3.11.0-5_h51639a9_openblas.conda + build_number: 5 + sha256: 620a6278f194dcabc7962277da6835b1e968e46ad0c8e757736255f5ddbfca8d + md5: bcc025e2bbaf8a92982d20863fe1fb69 + depends: + - libopenblas >=0.3.30,<0.3.31.0a0 + - libopenblas >=0.3.30,<1.0a0 + constrains: + - libcblas 3.11.0 5*_openblas + - liblapack 3.11.0 5*_openblas + - liblapacke 3.11.0 5*_openblas + - blas 2.305 openblas + - mkl <2026 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 18546 + timestamp: 1765819094137 +- conda: https://conda.anaconda.org/conda-forge/win-64/libblas-3.11.0-5_hf2e6a31_mkl.conda + build_number: 5 + sha256: f0cb7b2697461a306341f7ff32d5b361bb84f3e94478464c1e27ee01fc8f276b + md5: f9decf88743af85c9c9e05556a4c47c0 + depends: + - mkl >=2025.3.0,<2026.0a0 + constrains: + - liblapack 3.11.0 5*_mkl + - libcblas 3.11.0 5*_mkl + - blas 2.305 mkl + - liblapacke 3.11.0 5*_mkl + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 67438 + timestamp: 1765819100043 - conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb03c661_4.conda sha256: 2338a92d1de71f10c8cf70f7bb9775b0144a306d75c4812276749f54925612b6 md5: 1d29d2e33fe59954af82ef54a8af3fe1 @@ -3335,6 +3461,21 @@ packages: purls: [] size: 289680 timestamp: 1756599375485 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-5_h0358290_openblas.conda + build_number: 5 + sha256: 0cbdcc67901e02dc17f1d19e1f9170610bd828100dc207de4d5b6b8ad1ae7ad8 + md5: 6636a2b6f1a87572df2970d3ebc87cc0 + depends: + - libblas 3.11.0 5_h4a7cf45_openblas + constrains: + - liblapacke 3.11.0 5*_openblas + - blas 2.305 openblas + - liblapack 3.11.0 5*_openblas + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 18194 + timestamp: 1765818837135 - conda: https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-7_h8e06fc2_netlib.conda build_number: 7 sha256: 7940cc63673587cb7946831431b0527ce5707e24a54df87644c199e40c2714b4 @@ -3353,6 +3494,36 @@ packages: purls: [] size: 50122 timestamp: 1763440541127 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcblas-3.11.0-5_hb0561ab_openblas.conda + build_number: 5 + sha256: 38809c361bbd165ecf83f7f05fae9b791e1baa11e4447367f38ae1327f402fc0 + md5: efd8bd15ca56e9d01748a3beab8404eb + depends: + - libblas 3.11.0 5_h51639a9_openblas + constrains: + - liblapacke 3.11.0 5*_openblas + - liblapack 3.11.0 5*_openblas + - blas 2.305 openblas + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 18548 + timestamp: 1765819108956 +- conda: https://conda.anaconda.org/conda-forge/win-64/libcblas-3.11.0-5_h2a3cdd5_mkl.conda + build_number: 5 + sha256: 49dc59d8e58360920314b8d276dd80da7866a1484a9abae4ee2760bc68f3e68d + md5: b3fa8e8b55310ba8ef0060103afb02b5 + depends: + - libblas 3.11.0 5_hf2e6a31_mkl + constrains: + - liblapack 3.11.0 5*_mkl + - liblapacke 3.11.0 5*_mkl + - blas 2.305 mkl + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 68079 + timestamp: 1765819124349 - conda: https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2 sha256: fd1d153962764433fe6233f34a72cdeed5dcf8a883a85769e8295ce940b5b0c5 md5: c965a5aa0d5c1c37ffc62dff36e28400 @@ -3381,6 +3552,16 @@ packages: purls: [] size: 462942 timestamp: 1767821743793 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-h55c6f16_2.conda + sha256: 5fbeb2fc2673f0455af6079abf93faaf27f11a92574ad51565fa1ecac9a4e2aa + md5: 4cb5878bdb9ebfa65b7cdff5445087c5 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + purls: [] + size: 570068 + timestamp: 1770238262922 - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda sha256: d789471216e7aba3c184cd054ed61ce3f6dac6f87a50ec69291b9297f8c18724 md5: c277e0a4d549b03ac1e9d6cbbe3d017b @@ -3501,6 +3682,19 @@ packages: purls: [] size: 1040478 timestamp: 1770252533873 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgcc-15.2.0-hcbb3090_17.conda + sha256: 07ba27f2ef1ce444ce5c99d0f9590772fc5b58ba73c993477bfad74b17dfaa79 + md5: 65c07cee234440ae4d5d340fc4b2e69a + depends: + - _openmp_mutex + constrains: + - libgomp 15.2.0 17 + - libgcc-ng ==15.2.0=*_17 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 402928 + timestamp: 1770254186829 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_17.conda sha256: bdfe50501e4a2d904a5eae65a7ae26e2b7a29b473ab084ad55d96080b966502e md5: 1478bfa85224a65ab096d69ffd2af1e5 @@ -3523,6 +3717,18 @@ packages: purls: [] size: 27515 timestamp: 1770252591906 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-h07b0088_17.conda + sha256: 7b96f428cb932df8d7c1aa4e433ed29b779dd9571934afdf4f9093a85155a142 + md5: 45ba22eb5381fb602a45233d89ba27ae + depends: + - libgfortran5 15.2.0 hdae7583_17 + constrains: + - libgfortran-ng ==15.2.0=*_17 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 139757 + timestamp: 1770254394473 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-h68bc16d_17.conda sha256: b1c77b85da9a3e204de986f59e262268805c6a35dffdf3953f1b98407db2aef3 md5: 202fdf8cad9eea704c2b0d823d1732bf @@ -3536,6 +3742,18 @@ packages: purls: [] size: 2480824 timestamp: 1770252563579 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-hdae7583_17.conda + sha256: 9c41ff08f61c953cee13fc3df3c6245741e5a71e453b2c094a6d55b0eeda3669 + md5: c6329d871fb3207e9657c384128f5488 + depends: + - libgcc >=15.2.0 + constrains: + - libgfortran 15.2.0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + purls: [] + size: 599374 + timestamp: 1770254196706 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_17.conda sha256: b961b5dd9761907a7179678b58a69bb4fc16b940eb477f635aea3aec0a3f17a6 md5: 51b78c6a757575c0d12f4401ffc67029 @@ -3606,6 +3824,21 @@ packages: purls: [] size: 8349777 timestamp: 1761058442526 +- conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.2-default_h4379cf1_1000.conda + sha256: 8cdf11333a81085468d9aa536ebb155abd74adc293576f6013fc0c85a7a90da3 + md5: 3b576f6860f838f950c570f4433b086e + depends: + - libwinpthread >=12.0.0.r4.gg4f2fc60ca + - libxml2 + - libxml2-16 >=2.14.6 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 2411241 + timestamp: 1765104337762 - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda sha256: c467851a7312765447155e071752d7bf9bf44d610a5687e32706f480aad2833f md5: 915f5995e94f60e9a4826e0b0920ee88 @@ -3616,6 +3849,32 @@ packages: purls: [] size: 790176 timestamp: 1754908768807 +- conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda + sha256: 0dcdb1a5f01863ac4e8ba006a8b0dc1a02d2221ec3319b5915a1863254d7efa7 + md5: 64571d1dd6cdcfa25d0664a5950fdaa2 + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: LGPL-2.1-only + purls: [] + size: 696926 + timestamp: 1754909290005 +- conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-5_h47877c9_openblas.conda + build_number: 5 + sha256: c723b6599fcd4c6c75dee728359ef418307280fa3e2ee376e14e85e5bbdda053 + md5: b38076eb5c8e40d0106beda6f95d7609 + depends: + - libblas 3.11.0 5_h4a7cf45_openblas + constrains: + - blas 2.305 openblas + - liblapacke 3.11.0 5*_openblas + - libcblas 3.11.0 5*_openblas + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 18200 + timestamp: 1765818857876 - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-7_h8876d29_netlib.conda build_number: 7 sha256: 4de5b6aef4b2d42b4f71c6a3673118f99e323aed2ba2a66a3ed435b574010b1e @@ -3634,6 +3893,36 @@ packages: purls: [] size: 2901209 timestamp: 1763440547062 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.11.0-5_hd9741b5_openblas.conda + build_number: 5 + sha256: 735a6e6f7d7da6f718b6690b7c0a8ae4815afb89138aa5793abe78128e951dbb + md5: ca9d752201b7fa1225bca036ee300f2b + depends: + - libblas 3.11.0 5_h51639a9_openblas + constrains: + - libcblas 3.11.0 5*_openblas + - blas 2.305 openblas + - liblapacke 3.11.0 5*_openblas + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 18551 + timestamp: 1765819121855 +- conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.11.0-5_hf9ab0e9_mkl.conda + build_number: 5 + sha256: a2d33f5cc2b8a9042f2af6981c6733ab1a661463823eaa56595a9c58c0ab77e1 + md5: e62c42a4196dee97d20400612afcb2b1 + depends: + - libblas 3.11.0 5_hf2e6a31_mkl + constrains: + - libcblas 3.11.0 5*_mkl + - blas 2.305 mkl + - liblapacke 3.11.0 5*_mkl + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 80225 + timestamp: 1765819148014 - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda sha256: 755c55ebab181d678c12e49cced893598f2bab22d582fbbf4d8b83c18be207eb md5: c7c83eecbb72d88b940c249af56c8b17 @@ -3698,6 +3987,21 @@ packages: purls: [] size: 33731 timestamp: 1750274110928 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.30-pthreads_h94d23a6_4.conda + sha256: 199d79c237afb0d4780ccd2fbf829cea80743df60df4705202558675e07dd2c5 + md5: be43915efc66345cccb3c310b6ed0374 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + - libgfortran + - libgfortran5 >=14.3.0 + constrains: + - openblas >=0.3.30,<0.3.31.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 5927939 + timestamp: 1763114673331 - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.31-pthreads_h94d23a6_0.conda sha256: 166217a610185f9e22b3f4e0f80174d81240d6cfac8026b2f0158ff4f32b289a md5: 97ad7535866bf922275706c519b5c21d @@ -3713,6 +4017,21 @@ packages: purls: [] size: 5937816 timestamp: 1768555660623 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenblas-0.3.30-openmp_ha158390_4.conda + sha256: ebbbc089b70bcde87c4121a083c724330f02a690fb9d7c6cd18c30f1b12504fa + md5: a6f6d3a31bb29e48d37ce65de54e2df0 + depends: + - __osx >=11.0 + - libgfortran + - libgfortran5 >=14.3.0 + - llvm-openmp >=19.1.7 + constrains: + - openblas >=0.3.30,<0.3.31.0a0 + license: BSD-3-Clause + license_family: BSD + purls: [] + size: 4284132 + timestamp: 1768547079205 - conda: https://conda.anaconda.org/conda-forge/linux-64/libopentelemetry-cpp-1.21.0-hb9b0907_1.conda sha256: ba9b09066f9abae9b4c98ffedef444bbbf4c068a094f6c77d70ef6f006574563 md5: 1c0320794855f457dea27d35c4c71e23 @@ -3915,6 +4234,18 @@ packages: purls: [] size: 40311 timestamp: 1766271528534 +- conda: https://conda.anaconda.org/conda-forge/win-64/libwinpthread-12.0.0.r4.gg4f2fc60ca-h57928b3_10.conda + sha256: 0fccf2d17026255b6e10ace1f191d0a2a18f2d65088fd02430be17c701f8ffe0 + md5: 8a86073cf3b343b87d03f41790d8b4e5 + depends: + - ucrt + constrains: + - pthreads-win32 <0.0a0 + - msys2-conda-epoch <0.0a0 + license: MIT AND BSD-3-Clause-Clear + purls: [] + size: 36621 + timestamp: 1759768399557 - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda sha256: 6ae68e0b86423ef188196fff6207ed0c8195dd84273cb5623b85aa08033a410c md5: 5aa797f8787fe7a17d1b0821485b5adc @@ -3939,6 +4270,41 @@ packages: purls: [] size: 697033 timestamp: 1761766011241 +- conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-2.15.1-h779ef1b_1.conda + sha256: 8b47d5fb00a6ccc0f495d16787ab5f37a434d51965584d6000966252efecf56d + md5: 68dc154b8d415176c07b6995bd3a65d9 + depends: + - icu >=78.1,<79.0a0 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.1,<6.0a0 + - libxml2-16 2.15.1 h3cfd58e_1 + - libzlib >=1.3.1,<2.0a0 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: MIT + license_family: MIT + purls: [] + size: 43387 + timestamp: 1766327259710 +- conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-16-2.15.1-h3cfd58e_1.conda + sha256: a857e941156b7f462063e34e086d212c6ccbc1521ebdf75b9ed66bd90add57dc + md5: 07d73826fde28e7dbaec52a3297d7d26 + depends: + - icu >=78.1,<79.0a0 + - libiconv >=1.18,<2.0a0 + - liblzma >=5.8.1,<6.0a0 + - libzlib >=1.3.1,<2.0a0 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - libxml2 2.15.1 + license: MIT + license_family: MIT + purls: [] + size: 518964 + timestamp: 1766327232819 - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda sha256: d4bfe88d7cb447768e31650f06257995601f89076080e76df55e3112d4e47dc4 md5: edb0dca6bc32e4f4789199455a1dbeb8 @@ -3978,6 +4344,34 @@ packages: purls: [] size: 55476 timestamp: 1727963768015 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-21.1.8-h4a912ad_0.conda + sha256: 56bcd20a0a44ddd143b6ce605700fdf876bcf5c509adc50bf27e76673407a070 + md5: 206ad2df1b5550526e386087bef543c7 + depends: + - __osx >=11.0 + constrains: + - openmp 21.1.8|21.1.8.* + - intel-openmp <0.0a0 + license: Apache-2.0 WITH LLVM-exception + license_family: APACHE + purls: [] + size: 285974 + timestamp: 1765964756583 +- conda: https://conda.anaconda.org/conda-forge/win-64/llvm-openmp-21.1.8-h4fa8253_0.conda + sha256: 145c4370abe870f10987efa9fc15a8383f1dab09abbc9ad4ff15a55d45658f7b + md5: 0d8b425ac862bcf17e4b28802c9351cb + depends: + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + constrains: + - intel-openmp <0.0a0 + - openmp 21.1.8|21.1.8.* + license: Apache-2.0 WITH LLVM-exception + license_family: APACHE + purls: [] + size: 347566 + timestamp: 1765964942856 - conda: https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda sha256: 47326f811392a5fd3055f0f773036c392d26fdb32e4d8e7a8197eed951489346 md5: 9de5350a85c4a20c685259b889aa6393 @@ -4177,6 +4571,20 @@ packages: - pkg:pypi/mistune?source=hash-mapping size: 74250 timestamp: 1766504456031 +- conda: https://conda.anaconda.org/conda-forge/win-64/mkl-2025.3.0-hac47afa_455.conda + sha256: b2b4c84b95210760e4d12319416c60ab66e03674ccdcbd14aeb59f82ebb1318d + md5: fd05d1e894497b012d05a804232254ed + depends: + - llvm-openmp >=21.1.8 + - tbb >=2022.3.0 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: LicenseRef-IntelSimplifiedSoftwareOct2022 + license_family: Proprietary + purls: [] + size: 100224829 + timestamp: 1767634557029 - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl name: mpmath version: 1.3.0 @@ -4474,21 +4882,6 @@ packages: requires_dist: - numpy>=1.23.0 requires_python: '>=3.10' -- pypi: https://files.pythonhosted.org/packages/1b/46/6fa4ea94f1ddf969b2ee941290cca6f1bfac92b53c76ae5f44afe17ceb69/numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - name: numpy - version: 2.4.2 - sha256: c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32 - requires_python: '>=3.11' -- pypi: https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl - name: numpy - version: 2.4.2 - sha256: 7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1 - requires_python: '>=3.11' -- pypi: https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl - name: numpy - version: 2.4.2 - sha256: b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695 - requires_python: '>=3.11' - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py311h64a7726_0.conda sha256: 3f4365e11b28e244c95ba8579942b0802761ba7bb31c026f50d1a9ea9c728149 md5: a502d7aad449a1206efb366d6a12c52d @@ -4508,6 +4901,66 @@ packages: - pkg:pypi/numpy?source=hash-mapping size: 8065890 timestamp: 1707225944355 +- conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.4.2-py311h2e04523_1.conda + sha256: 2f9971a62316b9acb6ade749cebb59ffe750d1c2d99fe7061c6440589f6d3299 + md5: a8105076864776eceae69d64d30e24d7 + depends: + - python + - __glibc >=2.17,<3.0.a0 + - libstdcxx >=14 + - libgcc >=14 + - libblas >=3.9.0,<4.0a0 + - python_abi 3.11.* *_cp311 + - libcblas >=3.9.0,<4.0a0 + - liblapack >=3.9.0,<4.0a0 + constrains: + - numpy-base <0a0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/numpy?source=compressed-mapping + size: 9385101 + timestamp: 1770098496391 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/numpy-2.4.2-py311had1e860_1.conda + sha256: 09a06de7adea145124618b023e5b0da2949a7211083d0805c21960ab980e053b + md5: bebff6d1b28a10a57a586cc449688324 + depends: + - python + - __osx >=11.0 + - python 3.11.* *_cpython + - libcxx >=19 + - libblas >=3.9.0,<4.0a0 + - python_abi 3.11.* *_cp311 + - libcblas >=3.9.0,<4.0a0 + - liblapack >=3.9.0,<4.0a0 + constrains: + - numpy-base <0a0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/numpy?source=hash-mapping + size: 7451944 + timestamp: 1770098395802 +- conda: https://conda.anaconda.org/conda-forge/win-64/numpy-2.4.2-py311h80b3fa1_1.conda + sha256: c5cd26fb28d92d6c3843b96489f433ef87d1866d03a746f7228230b74bef431a + md5: a824c6667179120c458beb9e9394932f + depends: + - python + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - python_abi 3.11.* *_cp311 + - libcblas >=3.9.0,<4.0a0 + - liblapack >=3.9.0,<4.0a0 + - libblas >=3.9.0,<4.0a0 + constrains: + - numpy-base <0a0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/numpy?source=hash-mapping + size: 7803678 + timestamp: 1770098404597 - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl name: nvidia-cublas-cu12 version: 12.8.4.1 @@ -6198,6 +6651,50 @@ packages: - pkg:pypi/scipy?source=compressed-mapping size: 16967163 timestamp: 1768800888207 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/scipy-1.17.0-py311he9931d0_1.conda + sha256: d9f37c85cbf689be3672c8264eb81585ad8f6041a2fe545ec978f42e5da0202c + md5: 9c5c9dbdaf090ba8be3beb34c01495d0 + depends: + - __osx >=11.0 + - libblas >=3.9.0,<4.0a0 + - libcblas >=3.9.0,<4.0a0 + - libcxx >=19 + - libgfortran + - libgfortran5 >=14.3.0 + - liblapack >=3.9.0,<4.0a0 + - numpy <2.7 + - numpy >=1.23,<3 + - numpy >=1.25.2 + - python >=3.11,<3.12.0a0 + - python >=3.11,<3.12.0a0 *_cpython + - python_abi 3.11.* *_cp311 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/scipy?source=compressed-mapping + size: 14030449 + timestamp: 1768801949072 +- conda: https://conda.anaconda.org/conda-forge/win-64/scipy-1.17.0-py311h9c22a71_1.conda + sha256: c6896bbe8cb62b1743b86e4bae8c509233231412bf7ffd92bf0d5036a617dc8e + md5: 0d03c857517a5db3c1af5b553a528fac + depends: + - libblas >=3.9.0,<4.0a0 + - libcblas >=3.9.0,<4.0a0 + - liblapack >=3.9.0,<4.0a0 + - numpy <2.7 + - numpy >=1.23,<3 + - numpy >=1.25.2 + - python >=3.11,<3.12.0a0 + - python_abi 3.11.* *_cp311 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/scipy?source=hash-mapping + size: 14988880 + timestamp: 1768801728977 - conda: https://conda.anaconda.org/conda-forge/linux-64/scitokens-cpp-1.3.0-h096d96b_0.conda sha256: 11ad442837d2bd3c856c8a7ed08754ca430e6779999d898d1fa313fcd670458c md5: 946024dbdba971eeda33da76ae586694 @@ -6379,6 +6876,19 @@ packages: - blosc2>=2.3.0 - typing-extensions>=4.4.0 requires_python: '>=3.11' +- conda: https://conda.anaconda.org/conda-forge/win-64/tbb-2022.3.0-h3155e25_2.conda + sha256: abd9a489f059fba85c8ffa1abdaa4d515d6de6a3325238b8e81203b913cf65a9 + md5: 0f9817ffbe25f9e69ceba5ea70c52606 + depends: + - libhwloc >=2.12.2,<2.12.3.0a0 + - ucrt >=10.0.20348.0 + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + license: Apache-2.0 + license_family: APACHE + purls: [] + size: 155869 + timestamp: 1767886839029 - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda sha256: 6b6727a13d1ca6a23de5e6686500d0669081a117736a87c8abf444d60c1e40eb md5: 17b43cee5cc84969529d5d0b0309b2cb diff --git a/pyproject.toml b/pyproject.toml index adb445b..464be28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,8 @@ torchvision = { version = ">=0.20.1", index = "https://download.pytorch.org/whl/ python = ">=3.11,<3.12" omegaconf = ">=2.3.0,<3" hydra-core = ">=1.3.2,<2" +scipy = ">=1.17.0,<2" +debugpy = ">=1.8.20,<2" [tool.pixi.feature.fdp] platforms = ["linux-64"] diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index a6ddfa9..55f329b 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -4,9 +4,8 @@ def main(): hdf5_files = sorted( - Path( - "C:/Users/admin/PycharmProjects/FusionAIHub/scripts/training/" - ).glob("*_processed.h5") + Path("/scratch/gpfs/EKOLEMEN/foundation_model/" + ).glob("[0-9]*_processed.h5") ) # hdf5_files = sorted( @@ -14,11 +13,11 @@ def main(): # ) all_input_signals = [ - "mhr", "ece", "co2", "bes", # spectrograms - "gas", "ech", "pin", "tin", # actuators + "mhr", "ece", "co2", "bes", # spectrograms + "gas", "ech", "pin", "tin", # actuators "d_alpha", "mse", "ts_core_density", # diagnostics - "bolo", "irtv", "tangtv", # videos - # "text", # metadata + "bolo", "irtv", "tangtv", # videos + # "text", # metadata ] datasets = [ @@ -30,6 +29,7 @@ def main(): stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') + if __name__ == "__main__": # python scripts/data_preparation/make_processing_stats.py main() diff --git a/scripts/data_preparation/standardize_dataset.py b/scripts/data_preparation/standardize_dataset.py deleted file mode 100644 index 5f37a48..0000000 --- a/scripts/data_preparation/standardize_dataset.py +++ /dev/null @@ -1,24 +0,0 @@ -from pathlib import Path -from tokamak_foundation_model.data.data_loader import ( - TokamakH5Dataset, compute_preprocessing_stats) - -hdf5_files = sorted( - Path( - "C:/Users/admin/PycharmProjects/FusionAIHub/scripts/" - ).glob("*_processed.h5") -) -all_input_signals = [ - "mhr", "ece", "co2", # spectrograms - "gas", "ech", "pin", "tin", # actuators - "d_alpha", "mse", "ts_core_density", # diagnostics - "bolo", "irtv", "tangtv", # videos - "text", # metadata -] - -datasets = [ - TokamakH5Dataset( - hdf5_path=str(f), - input_signals=all_input_signals, - target_signals=all_input_signals, - ) for f in hdf5_files] -stats = compute_preprocessing_stats(datasets, '../preprocessing_stats.pt') diff --git a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml index caa712e..ede62a5 100644 --- a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml +++ b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml @@ -25,7 +25,7 @@ signals: input_ykey: block0_values source: default stft: true - sampling_rate: 500000 + sampling_rate: 10000 num_channels: 16 mse: @@ -34,7 +34,7 @@ signals: input_ykey: block0_values source: default stft: false - sampling_rate: 1000 + sampling_rate: 100 num_channels: 36 ts_core_density: @@ -43,8 +43,8 @@ signals: input_ykey: block0_values source: default stft: false - sampling_rate: 1000 - num_channels: 40 + sampling_rate: 100 + num_channels: 44 mhr: input_group: magnetics_high_resolution @@ -79,7 +79,7 @@ signals: input_ykey: block0_values source: default stft: false - sampling_rate: 1000 + sampling_rate: 10000 num_channels: 5 ech: @@ -88,7 +88,7 @@ signals: input_ykey: block0_values source: default stft: false - sampling_rate: 1000 + sampling_rate: 10000 num_channels: 11 pin: @@ -97,7 +97,7 @@ signals: input_ykey: block0_values source: default stft: false - sampling_rate: 1000 + sampling_rate: 10000 num_channels: 8 tin: @@ -106,7 +106,7 @@ signals: input_ykey: block0_values source: default stft: false - sampling_rate: 1000 + sampling_rate: 10000 num_channels: 8 bolo: @@ -115,7 +115,7 @@ signals: input_ykey: data source: video # reads from video_data_path/{shot}_image.h5 stft: false - sampling_rate: 1000 + sampling_rate: 50 num_channels: 48 # swap_axes: [0, 2] # swapaxes on ydata @@ -125,7 +125,7 @@ signals: input_ykey: data source: video stft: false - sampling_rate: 1000 + sampling_rate: 50 num_channels: 48 tangtv: @@ -134,5 +134,5 @@ signals: input_ykey: data source: video stft: false - sampling_rate: 1000 + sampling_rate: 50 num_channels: 48 \ No newline at end of file diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index c5428fe..e1ab704 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -1,5 +1,5 @@ import torch -from torch.utils.data import Dataset +from torch.utils.data import Dataset, DataLoader import numpy as np import h5py from pathlib import Path @@ -10,43 +10,171 @@ # TODO: implement this for calculation -class Welford: +class WelfordTensor: + """ + Welford algorithm for computing running statistics on batched multi-channel tensors. + + Computes per-channel statistics by aggregating across batch and all other dimensions. + + For signals (B, C, F, T) or (B, C, 1, T): computes stats per channel → shape (C,) + For profiles (B, S, T): computes stats per spatial point → shape (S,) + For videos (B, T, H, W): computes global stats → shape (1,) + """ + def __init__(self): - self.mean = 0 - self.std = 0 - self.min_val = 0 - self.max_val = 0 + self.mean = None + self.std = None + self.min_val = None + self.max_val = None self.n = 0 - self.M2 = 0 + self.M2 = None + self.initialized = False + + def _initialize(self, value: torch.Tensor): + """Initialize arrays based on first tensor's shape.""" + # Determine number of channels based on tensor shape (excluding batch dim) + if value.ndim == 4: + # (batch, channels, freq_bins, time) or (batch, channels, 1, time) + n_channels = value.shape[1] + elif value.ndim == 3: + # (batch, spatial_points, time) or (batch, time, height) - ambiguous + # Assume spatial/channel dim is second + n_channels = value.shape[1] + elif value.ndim == 2: + # (batch, time) - single channel + n_channels = 1 + else: + # Shouldn't happen, but treat as single channel + n_channels = 1 + + self.mean = torch.zeros(n_channels, dtype=torch.float64) + self.M2 = torch.zeros(n_channels, dtype=torch.float64) + self.min_val = torch.full((n_channels,), float('inf'), dtype=torch.float64) + self.max_val = torch.full((n_channels,), float('-inf'), dtype=torch.float64) + self.initialized = True - def update(self, value): + def update(self, value: torch.Tensor): + """ + Update statistics with new batched tensor. - if np.isnan(value): + Parameters + ---------- + value : torch.Tensor + Input tensor of shape: + - (batch, channels, freq_bins, time) for spectrograms + - (batch, channels, 1, time) for time series + - (batch, spatial_points, time) for profiles + - (batch, time, height, width) for videos + """ + # Skip if contains NaN + if torch.isnan(value).any(): return - self.n += 1 - delta = value - self.mean - self.mean += delta / self.n - delta2 = value - self.mean - self.M2 += delta * delta2 - self.min_val = min(self.min_val, value) - self.max_val = max(self.max_val, value) + # Initialize on first call + if not self.initialized: + self._initialize(value) + + # Convert to float64 for numerical stability + value = value.to(dtype=torch.float64) + + # Compute per-channel statistics by flattening batch and all non-channel dims + if value.ndim == 4 and value.shape[1] == self.mean.shape[0]: + # (batch, channels, freq_bins, time) → flatten batch, freq, time + # (B, C, F, T) → (C, B*F*T) + batch_size = value.shape[0] + n_channels = value.shape[1] + value_flat = value.permute(1, 0, 2, 3).reshape(n_channels, -1) # (C, B*F*T) + + # Per-channel mean, min, max + batch_mean = value_flat.mean(dim=1) + batch_min = value_flat.min(dim=1).values + batch_max = value_flat.max(dim=1).values + n_samples = value_flat.shape[1] + + # For variance, we need sum of squared deviations + batch_var = value_flat.var(dim=1, unbiased=False) + batch_M2 = batch_var * n_samples + + elif value.ndim == 3: + # (batch, spatial_points, time) → flatten batch, time + # (B, S, T) → (S, B*T) + n_channels = value.shape[1] + value_flat = value.permute(1, 0, 2).reshape(n_channels, -1) # (S, B*T) + + batch_mean = value_flat.mean(dim=1) + batch_min = value_flat.min(dim=1).values + batch_max = value_flat.max(dim=1).values + n_samples = value_flat.shape[1] + + batch_var = value_flat.var(dim=1, unbiased=False) + batch_M2 = batch_var * n_samples + + else: + # Video (batch, time, height, width) → global statistics + value_flat = value.flatten() + + batch_mean = torch.tensor([value_flat.mean()], dtype=torch.float64) + batch_min = torch.tensor([value_flat.min()], dtype=torch.float64) + batch_max = torch.tensor([value_flat.max()], dtype=torch.float64) + n_samples = value_flat.shape[0] + + batch_var = value_flat.var(unbiased=False) + batch_M2 = batch_var * n_samples + + # Parallel Welford's algorithm for combining batches + # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm + n_old = self.n + n_new = n_samples + n_total = n_old + n_new + + # Update mean + delta = batch_mean - self.mean + self.mean = (n_old * self.mean + n_new * batch_mean) / n_total + + # Update M2 (sum of squared deviations) + # M2_total = M2_old + M2_new + delta^2 * n_old * n_new / n_total + self.M2 = self.M2 + batch_M2 + delta * delta * n_old * n_new / n_total + + self.n = n_total + + # Update min/max + self.min_val = torch.minimum(self.min_val, batch_min) + self.max_val = torch.maximum(self.max_val, batch_max) def _compute_std(self): - self.std = np.sqrt(self.M2 / (self.n - 1 + 1e-8)) + """Compute standard deviation from M2.""" + if self.n > 1: + self.std = torch.sqrt(self.M2 / (self.n - 1)) + else: + self.std = torch.zeros_like(self.mean) def compute(self): + """ + Compute final statistics. + + Returns + ------- + dict + Dictionary with numpy arrays: + - 'mean': per-channel mean + - 'std': per-channel standard deviation + - 'min_val': per-channel minimum + - 'max_val': per-channel maximum + """ self._compute_std() + return { - "mean": self.mean, - "std": self.std, - "min_val": self.min_val, - "max_val": self.max_val, + "mean": self.mean.numpy(), + "std": self.std.numpy(), + "min_val": self.min_val.numpy(), + "max_val": self.max_val.numpy(), } def compute_preprocessing_stats( - datasets, output_path="preprocessing_stats.pt", num_samples=1000 + datasets, + output_path="preprocessing_stats.pt", + num_samples=1000 ): """Compute preprocessing statistics across multiple datasets. @@ -59,60 +187,28 @@ def compute_preprocessing_stats( from tqdm import tqdm combined = ConcatDataset(datasets) - stats = {} + dataloader = DataLoader(combined, batch_size=32, collate_fn=collate_fn, num_workers=1) # Get signal names from first dataset signal_configs = datasets[0].SIGNAL_CONFIGS + movie_configs = datasets[0].MOVIE_CONFIGS - for config in signal_configs: - print(f"Computing statistics for {config.name}...") - - # Collect values - values = [] - - for idx in tqdm(range(len(combined))): - batch = combined[int(idx)] - if config.name in batch['inputs']: - values.append(batch['inputs'][config.name]) - values.append(batch['targets'][config.name]) - - if not values: - continue + welford_stats = {cfg.name: WelfordTensor() for cfg in signal_configs + movie_configs} - # Stack and compute statistics - if values[0].ndim == 2: - all_values = torch.cat(values, dim=1) # (channels, time) - elif values[0].ndim == 3: - all_values = torch.cat(values, dim=2) # (channels, freq_bins, time) - else: - raise ValueError(f"Invalid tensor shape: {values[0].shape}") - - # Compute per-channel statistics - # Reduce over all dimensions except channel dimension (dim=1) - dims_to_reduce = list(range(all_values.ndim)) - dims_to_reduce.remove(0) # Keep channel dimension - - valid_mask = ~torch.isnan(all_values) - - # For mean/std: use nanmean + manual std - mean = all_values.nanmean(dim=dims_to_reduce) - mean_expanded = mean.view(-1, *([1] * (all_values.ndim - 1))) - std = ((all_values - mean_expanded) ** 2).nanmean(dim=dims_to_reduce).sqrt() - - # For min/max: mask out NaNs with inf - min_val = all_values.nan_to_num(posinf=float("inf"), nan=float("inf")).min() - max_val = all_values.nan_to_num(neginf=float("-inf"), nan=float("-inf")).max() + for batch in tqdm(dataloader): + for modality_name, tensor in batch.items(): + # Update statistics + welford_stats[modality_name].update(tensor) - stats[config.name] = { - "mean": mean, - "std": std, - "min_val": min_val.item(), - "max_val": max_val.item(), - } + # Compute final statistics + final_stats = { + modality: tracker.compute() + for modality, tracker in welford_stats.items() + } + torch.save(final_stats, output_path) - torch.save(stats, output_path) print(f"Saved statistics to {output_path}") - return stats + return final_stats @dataclass @@ -241,6 +337,7 @@ class TokamakH5Dataset(Dataset): apply_stft=False, preprocess=PreprocessConfig(method="none"), ), + # TODO: Include Gas as additional actuator!!! SignalConfig( "mse", ["mse"], @@ -266,16 +363,16 @@ class TokamakH5Dataset(Dataset): ] def __init__( - self, - hdf5_path: str, - chunk_duration_s: float = 0.5, - n_fft: int = 1024, - hop_length: int = 256, - preprocessing_stats: Optional[dict] = None, - prediction_mode: bool = True, - prediction_horizon_s: float = 0.2, - input_signals: Optional[list[str]] = None, - target_signals: Optional[list[str]] = None, + self, + hdf5_path: str, + chunk_duration_s: float = 0.5, + n_fft: int = 1024, + hop_length: int = 256, + preprocessing_stats: Optional[dict] = None, + prediction_mode: bool = False, + prediction_horizon_s: float = 0.2, + input_signals: Optional[list[str]] = None, + target_signals: Optional[list[str]] = None, ): # Make instance-level copies to avoid class-level mutation self.signal_configs = copy.deepcopy(self.SIGNAL_CONFIGS) @@ -298,10 +395,12 @@ def __init__( self._update_preprocessing_stats() self.h5_file = None - - with h5py.File(self.hdf5_path, "r") as f: - self.duration, self.t0_indices = self._compute_duration_and_t0_indices(f) - + try: + with h5py.File(self.hdf5_path, "r") as f: + self.duration, self.t0_indices = self._compute_duration_and_t0_indices(f) + except OSError as e: + print(self.hdf5_path) + raise e # In prediction mode, reduce length to ensure extended window fits if self.prediction_mode: total_window = self.chunk_duration_s + self.prediction_horizon_s @@ -552,7 +651,11 @@ def _open_hdf5(self): ) def _load_signal_raw( - self, f: h5py.File, config: SignalConfig, t_start: float, t_end: float + self, + f: h5py.File, + config: SignalConfig, + t_start: float, + t_end: float ) -> torch.Tensor: """ Load raw signal at native sampling rate within time window. @@ -575,17 +678,7 @@ def _load_signal_raw( """ duration_s = t_end - t_start - # Step 1: Check if signal has data after t=0 - if config.name not in self.t0_indices: - return torch.zeros( - (round(duration_s * config.target_fs), config.num_channels) - ) - - t0_info = self.t0_indices[config.name] - t0_idx = t0_info["index"] - t0_time_s = t0_info["time_s"] - - # Step 2: Find the signal in HDF5 + # Find the signal in HDF5 data_group = None for key_path in config.hdf5_keys: try: @@ -606,48 +699,44 @@ def _load_signal_raw( ydata_ds = data_group["ydata"] xdata_ds = data_group["xdata"] - # Load first and last timestamp to compute sampling rate - t_first = xdata_ds[0] / 1000.0 - t_last = xdata_ds[-1] / 1000.0 + # Get time range and sample count + xdata_start_s = xdata_ds[0] / 1000.0 + xdata_end_s = xdata_ds[-1] / 1000.0 n_samples = xdata_ds.shape[0] - if n_samples < 2 or t_last == t_first: + if n_samples < 2 or xdata_end_s == xdata_start_s: return torch.zeros( (round(duration_s * config.target_fs), config.num_channels) ) - fs_raw = (n_samples - 1) / (t_last - t_first) + # Compute actual sampling frequency from the data + actual_fs = (n_samples - 1) / (xdata_end_s - xdata_start_s) - # Step 3: Initialize output with zeros at raw sampling rate + # Step 1: Initialize output array with zeros output = np.zeros( - (round(duration_s * fs_raw), config.num_channels), dtype=np.float32 + (round(duration_s * actual_fs), config.num_channels), + dtype=np.float32 ) - # Step 4: Calculate HDF5 indices for requested time range - # xdata[t0_idx] = t0_time_s (actual time, e.g., 0.005s if first sample is at 5ms) - # To find data at user's t_start: - # We want: xdata[i] ≈ t_start - # We know: xdata[i] ≈ t0_time_s + (i - t0_idx) / fs_raw - # Solving: i ≈ t0_idx + (t_start - t0_time_s) * fs_raw - hdf5_start = t0_idx + round((t_start - t0_time_s) * fs_raw) - hdf5_end = t0_idx + round((t_end - t0_time_s) * fs_raw) - - # Clamp to valid HDF5 range - hdf5_start = max(0, min(hdf5_start, n_samples)) - hdf5_end = max(0, min(hdf5_end, n_samples)) - - # Step 5: If there's data to load - if hdf5_start < hdf5_end: - # Load from HDF5 - data = ydata_ds[hdf5_start:hdf5_end] - np.nan_to_num(data, copy=False, nan=0.0) + # Step 2: Calculate which HDF5 indices correspond to [t_start, t_end] + # xdata[i] = xdata_start_s + i / actual_fs + # Solving for i: i = (t - xdata_start_s) * actual_fs + hdf5_start = round((t_start - xdata_start_s) * actual_fs) + hdf5_end = round((t_end - xdata_start_s) * actual_fs) + + # Clamp to valid HDF5 range [0, n_samples] + hdf5_start_clamped = max(0, min(hdf5_start, n_samples)) + hdf5_end_clamped = max(0, min(hdf5_end, n_samples)) - # Calculate what time range the loaded data represents - # xdata[hdf5_start] ≈ t0_time_s + (hdf5_start - t0_idx) / fs_raw - loaded_t_start = t0_time_s + (hdf5_start - t0_idx) / fs_raw + # Step 3: Load data if there's any overlap + if hdf5_start_clamped < hdf5_end_clamped: + data = ydata_ds[hdf5_start_clamped:hdf5_end_clamped] + np.nan_to_num(data, copy=False, nan=0.0) - # Position in output (which represents [t_start, t_end]) - output_start = round((loaded_t_start - t_start) * fs_raw) + # Step 4: Calculate where to insert in output array + # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fs + # This corresponds to output index: (that_time - t_start) * actual_fs + output_start = hdf5_start_clamped - hdf5_start output_end = output_start + data.shape[0] # Clamp to output bounds @@ -661,9 +750,14 @@ def _load_signal_raw( src_end -= output_end - output.shape[0] output_end = output.shape[0] - # Copy data to output + # Insert data into output if src_start < src_end and output_start < output_end: - output[output_start:output_end] = data[src_start:src_end] + if data.shape[1] == config.num_channels: + output[output_start:output_end] = data[src_start:src_end] + elif data.shape[1] > config.num_channels: + output[output_start:output_end] = data[src_start:src_end, :config.num_channels] + else: + output[output_start:output_end, :data.shape[1]] = data[src_start:src_end] # Step 6: Convert to tensor and resample to target frequency tensor = torch.from_numpy(output).float() @@ -757,14 +851,20 @@ def _process_signal( return processed def _load_movie_raw( - self, f: h5py.File, config: MovieConfig, t_start: float, t_end: float + self, + f: h5py.File, + config: MovieConfig, + t_start: float, + t_end: float ) -> torch.Tensor: """Load raw movie data without resampling (for prediction mode). Returns: Raw movie array at native frame rate, shape (time, height, width) """ - # Try to find the movie in HDF5 + duration_s = t_end - t_start + + # Find the movie in HDF5 data_group = None for key_path in config.hdf5_keys: try: @@ -776,72 +876,88 @@ def _load_movie_raw( break except KeyError: continue - - # Extract data with time slicing + ydata_ds = data_group["ydata"] xdata_ds = data_group["xdata"] - # Load only first and last timestamp - t0 = xdata_ds[0] / 1000.0 - t1 = xdata_ds[-1] / 1000.0 - n_samples = xdata_ds.shape[0] + if ydata_ds.size == 0: + return torch.zeros( + (round(duration_s * config.target_fps), config.height, config.width) + ) - fps_raw = (n_samples - 1) / (t1 - t0) - duration_s = t_end - t_start + # Get time range and frame count + xdata_start_s = xdata_ds[0] / 1000.0 + xdata_end_s = xdata_ds[-1] / 1000.0 + n_frames = xdata_ds.shape[0] + + if n_frames < 2 or xdata_end_s == xdata_start_s: + return torch.zeros( + (round(duration_s * config.target_fps), config.height, config.width) + ) - if n_samples < 2 or t1 == t0: - n_frames = round(duration_s * config.target_fps) - return torch.zeros(max(n_frames, 1), config.height, config.width) + # Compute actual frame rate from the data + actual_fps = (n_frames - 1) / (xdata_end_s - xdata_start_s) + # Get actual dimensions from data raw_height, raw_width = ydata_ds.shape[1], ydata_ds.shape[2] - ydata = np.zeros( - (max(1, round(duration_s * fps_raw)), raw_height, raw_width), dtype=np.float32 + + # Step 1: Initialize output array with zeros at actual fps + output = np.zeros( + (round(duration_s * actual_fps), raw_height, raw_width), + dtype=np.float32 ) - - # Compute indices directly (no full xdata load) - start_idx = max(0, int((t_start - t0) * fps_raw)) - end_idx = min(n_samples, int((t_end - t0) * fps_raw)) - if end_idx > start_idx: - data = ydata_ds[start_idx:end_idx] + # Step 2: Calculate which HDF5 indices correspond to [t_start, t_end] + # xdata[i] = xdata_start_s + i / actual_fps + # Solving for i: i = (t - xdata_start_s) * actual_fps + hdf5_start = round((t_start - xdata_start_s) * actual_fps) + hdf5_end = round((t_end - xdata_start_s) * actual_fps) + + # Clamp to valid HDF5 range [0, n_frames] + hdf5_start_clamped = max(0, min(hdf5_start, n_frames)) + hdf5_end_clamped = max(0, min(hdf5_end, n_frames)) + + # Step 3: Load data if there's any overlap + if hdf5_start_clamped < hdf5_end_clamped: + data = ydata_ds[hdf5_start_clamped:hdf5_end_clamped] data[np.isnan(data)] = 0.0 - # Compute offset based on actual start time - actual_t_start = t0 + start_idx / fps_raw - idx_1 = round((actual_t_start - t_start) * fps_raw) - idx_2 = idx_1 + data.shape[0] - # Clamp to array bounds + # Step 4: Calculate where to insert in output array + # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fps + # This corresponds to output index: (that_time - t_start) * actual_fps + output_start = hdf5_start_clamped - hdf5_start + output_end = output_start + data.shape[0] + + # Clamp to output bounds src_start = 0 src_end = data.shape[0] - if idx_1 < 0: - src_start = -idx_1 - idx_1 = 0 - if idx_2 > ydata.shape[0]: - src_end -= idx_2 - ydata.shape[0] - idx_2 = ydata.shape[0] + if output_start < 0: + src_start = -output_start + output_start = 0 + if output_end > output.shape[0]: + src_end -= output_end - output.shape[0] + output_end = output.shape[0] - if (idx_1 == 0 and idx_2 == ydata.shape[0] and - src_start == 0 and src_end == data.shape[0]): - ydata = data # No copy needed - else: - ydata[idx_1:idx_2] = data[src_start:src_end] + # Insert data into output + if src_start < src_end and output_start < output_end: + output[output_start:output_end] = data[src_start:src_end] - tensor = torch.from_numpy(ydata).float() + # Step 5: Convert to tensor and resample to target fps and dimensions + tensor = torch.from_numpy(output).float() + # Resample using trilinear interpolation + # Input: (time, height, width) → add batch and channel dims + # Output: (batch=1, channels=1, time, height, width) tensor = ( - F.interpolate( - tensor.unsqueeze(0).unsqueeze(0), - size=( - round(duration_s * config.target_fps), - config.height, - config.width, - ), - mode="trilinear", - align_corners=False, - ) - .squeeze(0) - .squeeze(0) + F.interpolate(tensor.unsqueeze(0).unsqueeze(0), + size=(round(duration_s * config.target_fps), + config.height, + config.width, + ), + mode="trilinear", + align_corners=False, + ).squeeze(0).squeeze(0) ) return tensor @@ -853,7 +969,7 @@ def __getitem__(self, idx): return self._getitem_prediction(idx) else: return self._getitem_standard(idx) - + def _getitem_standard(self, idx): """Original __getitem__ logic.""" t_start = idx * self.chunk_duration_s diff --git a/src/tokamak_foundation_model/data/prepare_data.py b/src/tokamak_foundation_model/data/prepare_data.py index 892c47c..a53b95d 100644 --- a/src/tokamak_foundation_model/data/prepare_data.py +++ b/src/tokamak_foundation_model/data/prepare_data.py @@ -7,6 +7,9 @@ from omegaconf import DictConfig, OmegaConf from pathlib import Path from tqdm.auto import tqdm +from scipy.interpolate import interp1d +import os + log = logging.getLogger(__name__) @@ -14,10 +17,83 @@ _VIDEO_DATA_PATH = Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_image_data") +def _resample_time_series(data, time, target_frequency): + """ + Resample non-uniformly sampled time series to uniform sampling. + + Parameters: + ----------- + data : np.ndarray, shape (n_samples, ...) + Time series data + time : np.ndarray, shape (n_samples,) + Time axis (can be non-uniform) + target_frequency : float + Desired sampling frequency in Hz + + Returns: + -------- + resampled_data : np.ndarray + Uniformly resampled data + new_time : np.ndarray + New uniform time axis + """ + if len(data) <= 1: + return time.copy(), data.copy() + + # Calculate target sampling period + dt = 1.0 / target_frequency + + # Create uniform time grid + n_samples = int(np.ceil((time[-1] - time[0]) / dt)) + 1 + new_time = time[0] + np.arange(n_samples) * dt + + # Handle multi-dimensional data + original_shape = data.shape + if data.ndim > 1: + # Flatten all dimensions except the first (time) + data_flat = data.reshape(data.shape[0], -1) + resampled_flat = np.full((len(new_time), data_flat.shape[1]), np.nan) + + # Interpolate each channel, handling NaNs + for i in range(data_flat.shape[1]): + # Find valid (non-NaN) data points + valid_mask = ~np.isnan(data_flat[:, i]) + + if np.sum(valid_mask) >= 2: # Need at least 2 points to interpolate + valid_time = time[valid_mask] + valid_data = data_flat[valid_mask, i] + + # Only interpolate within the range of valid data + interpolator = interp1d(valid_time, valid_data, kind='linear', + bounds_error=False, fill_value=np.nan) + resampled_flat[:, i] = interpolator(new_time) + # else: remains NaN (initialized above) + + # Reshape back to original dimensions (except time axis) + new_shape = (len(new_time),) + original_shape[1:] + resampled_data = resampled_flat.reshape(new_shape) + else: + # 1D case + valid_mask = ~np.isnan(data) + + if np.sum(valid_mask) >= 2: + valid_time = time[valid_mask] + valid_data = data[valid_mask] + + interpolator = interp1d(valid_time, valid_data, kind='linear', + bounds_error=False, fill_value=np.nan) + resampled_data = interpolator(new_time) + else: + # Not enough valid data to interpolate + resampled_data = np.full(len(new_time), np.nan) + + return new_time, resampled_data + + def _get_valid_shots( - shot_list: list[int], - input_data_path: Path, - video_data_path: Path, + shot_list: list[int], + input_data_path: Path, + video_data_path: Path, ) -> list[int]: """Return only shots that have files in *both* the main data path and the video data path. Expects ``{shot}.h5`` in input_data_path and @@ -39,8 +115,8 @@ def _get_valid_shots( n_missing = len(requested) - len(valid) if n_missing: log.warning( - f"{n_missing}/{len(requested)} requested shots missing from one or " - f"both data paths – skipped" + f"{n_missing}/{len(requested)} requested shots missing from one " + f"or both data paths – skipped" ) log.info(f"{len(valid)} shots available in both paths") return valid @@ -58,7 +134,8 @@ def _process_shot(shot: int, cfg_dict: dict) -> str | None: """ try: input_data_path = Path(cfg_dict["input_data_path"]) - video_data_path = Path(cfg_dict.get("video_data_path", str(_VIDEO_DATA_PATH))) + video_data_path = Path( + cfg_dict.get("video_data_path", str(_VIDEO_DATA_PATH))) output_data_path = Path(cfg_dict["output_data_path"]) output_data_path.mkdir(parents=True, exist_ok=True) @@ -98,7 +175,12 @@ def _process_shot(shot: int, cfg_dict: dict) -> str | None: if sig_cfg.get("swap_axes") is not None: ydata = ydata.swapaxes(*sig_cfg["swap_axes"]) - read_data[abbr] = (xdata, ydata) + xdata, ydata = _resample_time_series( + data=ydata, + time=xdata / 1000, + target_frequency=sig_cfg["sampling_rate"]) + + read_data[abbr] = (xdata * 1000, ydata) if not read_data: return f"shot {shot}: no data read – skipped" @@ -107,12 +189,14 @@ def _process_shot(shot: int, cfg_dict: dict) -> str | None: with h5py.File(output_file, "w") as f: for abbr, (xdata, ydata) in read_data.items(): grp = f.create_group(abbr) - grp.create_dataset("xdata", data=xdata) - grp.create_dataset("ydata", data=ydata) + grp.create_dataset("xdata", data=xdata, dtype='f8') + grp.create_dataset("ydata", data=ydata, dtype='f8') + os.chmod(output_file, 0o664) return None # success except Exception as e: + log.info(f"shot {shot}: {type(e).__name__}: {e}") return f"shot {shot}: {type(e).__name__}: {e}" @@ -122,7 +206,8 @@ def main(cfg: DictConfig) -> None: mod_cfg = cfg.modalities input_data_path = Path(mod_cfg.input_data_path) - video_data_path = Path(mod_cfg.get("video_data_path", str(_VIDEO_DATA_PATH))) + video_data_path = Path( + mod_cfg.get("video_data_path", str(_VIDEO_DATA_PATH))) num_workers = mod_cfg.get("num_workers", 8) # ── filter to shots that exist in both paths ── @@ -144,9 +229,10 @@ def main(cfg: DictConfig) -> None: worker = partial(_process_shot, cfg_dict=cfg_dict) errors = [] - + with Pool(processes=num_workers) as pool: - for i, err in enumerate(tqdm(pool.imap_unordered(worker, shots), total=len(shots))): + for i, err in enumerate( + tqdm(pool.imap_unordered(worker, shots), total=len(shots))): if err is not None: log.error(err) errors.append(err) @@ -158,5 +244,4 @@ def main(cfg: DictConfig) -> None: if __name__ == "__main__": - # python -m tokamak_foundation_model.data.prepare_data - main() \ No newline at end of file + main() diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index 3e993df..24573ad 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -14,13 +14,13 @@ class MultimodalTrainer: def __init__( - self, - model: nn.Module, - optimizer: optim.Optimizer, - loss_fn: nn.Module, - device: torch.device, - epochs: int, - checkpoint_path: str | Path = "checkpoint.pth", + self, + model: nn.Module, + optimizer: optim.Optimizer, + loss_fn: nn.Module, + device: torch.device, + epochs: int, + checkpoint_path: str | Path = "checkpoint.pth", ): self.model = model self.optimizer = optimizer @@ -52,7 +52,8 @@ def _train_epoch(self, dataloader: DataLoader): total_loss += loss.item() if batch_idx % 10 == 0: - print(f" Batch {batch_idx}/{len(dataloader)}, Loss: {loss.item():.4f}") + print(f" Batch {batch_idx}/{len(dataloader)}," + f" Loss: {loss.item():.4f}") return total_loss / len(dataloader) def _validate_epoch(self, dataloader: DataLoader): @@ -72,7 +73,11 @@ def _validate_epoch(self, dataloader: DataLoader): total_loss += loss.item() return total_loss / len(dataloader) - def train(self, train_dataloader: DataLoader, val_dataloader: DataLoader = None): + def train( + self, + train_dataloader: DataLoader, + val_dataloader: DataLoader = None + ): best_val_loss = float("inf") for epoch in range(self.epochs): print(f"Epoch {epoch + 1}/{self.epochs}") @@ -94,7 +99,8 @@ def train(self, train_dataloader: DataLoader, val_dataloader: DataLoader = None) def load_checkpoint(self, checkpoint_path=None): path = checkpoint_path if checkpoint_path else self.checkpoint_path if os.path.exists(path): - self.model.load_state_dict(torch.load(path, map_location=self.device)) + self.model.load_state_dict(torch.load( + path, map_location=self.device)) print(f"Model loaded from checkpoint: {path}") else: print(f"No checkpoint found at: {path}") @@ -102,16 +108,16 @@ def load_checkpoint(self, checkpoint_path=None): class UnimodalTrainer: def __init__( - self, - model: nn.Module, - optimizer: optim.Optimizer, - loss_fn: nn.Module, - device: torch.device, - epochs: int, - lr_scheduler: optim.lr_scheduler.LRScheduler | None = None, - log_interval: int | None = None, - drawer: object | None = None, - checkpoint_path: str | Path = "checkpoint.pth", + self, + model: nn.Module, + optimizer: optim.Optimizer, + loss_fn: nn.Module, + device: torch.device, + epochs: int, + lr_scheduler: optim.lr_scheduler.LRScheduler | None = None, + log_interval: int | None = None, + drawer: object | None = None, + checkpoint_path: str | Path = "checkpoint.pth", ): self.model = model self.optimizer = optimizer @@ -127,10 +133,10 @@ def __init__( self.best_checkpoint_path = p.with_name(p.stem + "_best" + p.suffix) def _log_epoch( - self, - epoch: int, - train_loss: float, - val_loss: float = 0, + self, + epoch: int, + train_loss: float, + val_loss: float = 0, ): logger.info( f"Epoch {epoch + 1}/{self.epochs}," @@ -142,9 +148,9 @@ def _log_epoch( self.drawer(self.model, epoch, train_loss, val_loss) def _train_epoch( - self, - dataloader: DataLoader, - modality_key: str, + self, + dataloader: DataLoader, + modality_key: str, ): self.model.train() total_loss = 0 @@ -159,9 +165,9 @@ def _train_epoch( return total_loss / len(dataloader) def _validate_epoch( - self, - dataloader: DataLoader, - modality_key: str, + self, + dataloader: DataLoader, + modality_key: str, ): self.model.eval() total_loss = 0 @@ -174,10 +180,10 @@ def _validate_epoch( return total_loss / len(dataloader) def train( - self, - train_dataloader: DataLoader, - val_dataloader: DataLoader = None, - modality_key: str = "dalpha", + self, + train_dataloader: DataLoader, + val_dataloader: DataLoader = None, + modality_key: str = "dalpha", ): # Setup Training Loop self._current_epoch = 0 @@ -185,7 +191,8 @@ def train( best_val_loss = float("inf") if self.drawer: self.drawing_path = Path(self.checkpoint_path).parent / "plots" - self.drawer.setup(train_dataloader, self.drawing_path, modality_key) + self.drawer.setup( + train_dataloader, self.drawing_path, modality_key) # Train for epoch in range(self.epochs): @@ -212,7 +219,15 @@ def train( logger.info(f" Validation Loss: {val_loss:.4f}") if val_loss < best_val_loss: best_val_loss = val_loss - torch.save(self.model.state_dict(), self.best_checkpoint_path) + torch.save({ + "model": self.model, + "optimizer_state_dict": self.optimizer.state_dict(), + "scheduler_state_dict": self.lr_scheduler.state_dict(), + "epoch": epoch, + "loss": train_loss, + }, + self.best_checkpoint_path, + ) logger.info( f" Best validation loss: {best_val_loss:.4f}, " f"best model checkpoint saved!" @@ -228,12 +243,11 @@ def train( logger.info("Training complete.") def load_checkpoint(self, checkpoint_path=None): - """ - TODO: Modify this as we have more information stored in the checkpoint now. - """ path = checkpoint_path if checkpoint_path else self.checkpoint_path if os.path.exists(path): - self.model.load_state_dict(torch.load(path, map_location=self.device)) + checkpoint = torch.load( + path, weights_only=False, map_location=self.device) + self.model = checkpoint["model"] print(f"Model loaded from checkpoint: {path}") else: print(f"No checkpoint found at: {path}") \ No newline at end of file From fe38d8cb1ea215086bfd90ac8b672e0f0ce32a44 Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 24 Feb 2026 14:36:40 -0500 Subject: [PATCH 18/30] Added scripts for data fetching in Omega. TODO: Write a documentation. --- scripts/data_fetching_omega/config_atlas.yaml | 1848 +++++++++++++++++ .../data_fetching_omega/config_chiron.yaml | 19 + scripts/data_fetching_omega/read_mds.sh | 251 +++ .../submit_read_mds_batches.sh | 196 ++ 4 files changed, 2314 insertions(+) create mode 100644 scripts/data_fetching_omega/config_atlas.yaml create mode 100644 scripts/data_fetching_omega/config_chiron.yaml create mode 100644 scripts/data_fetching_omega/read_mds.sh create mode 100644 scripts/data_fetching_omega/submit_read_mds_batches.sh diff --git a/scripts/data_fetching_omega/config_atlas.yaml b/scripts/data_fetching_omega/config_atlas.yaml new file mode 100644 index 0000000..76a536d --- /dev/null +++ b/scripts/data_fetching_omega/config_atlas.yaml @@ -0,0 +1,1848 @@ +trees: + d3d: + - \D3D::TOP.ELECTRONS.TS.BLESSED.CORE:DENSITY + - \D3D::TOP.ELECTRONS.TS.BLESSED.CORE:TEMP + - \D3D::TOP.ELECTRONS.TS.BLESSED.TANGENTIAL:DENSITY + - \D3D::TOP.ELECTRONS.TS.BLESSED.TANGENTIAL:TEMP + - \D3D::TOP.ELECTRONS.TS.BLESSED.DIVERTOR:DENSITY + - \D3D::TOP.ELECTRONS.TS.BLESSED.DIVERTOR:TEMP + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF01 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF02 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF03 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF04 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF05 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF06 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF07 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF08 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF09 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF10 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF11 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF12 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF13 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF14 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF15 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF16 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF17 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF18 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF19 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF20 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF21 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF22 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF23 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF24 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF25 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF26 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF27 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF28 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF29 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF30 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF31 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF32 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF33 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF34 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF35 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF36 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF37 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF38 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF39 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF40 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF41 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF42 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF43 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF44 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF45 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF46 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF47 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF48 + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL01:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL02:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL03:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL04:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL05:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL06:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL07:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL08:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL09:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL10:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL11:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL12:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL13:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL14:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL15:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL16:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL17:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL18:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL19:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL20:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL21:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL22:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL23:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL24:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL25:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL26:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL27:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL28:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL29:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL30:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL31:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL32:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL33:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL34:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL35:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL36:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL37:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL38:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL39:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL40:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL41:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL42:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL43:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL44:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL45:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL46:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL47:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL48:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL01:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL02:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL03:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL04:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL05:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL06:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL07:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL08:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL09:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL10:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL11:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL12:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL13:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL14:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL15:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL16:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL17:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL18:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL19:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL20:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL21:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL22:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL23:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL24:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL25:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL26:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL27:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL28:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL29:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL30:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL31:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL32:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL33:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL34:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL35:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL36:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL37:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL38:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL39:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL40:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL41:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL42:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL43:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL44:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL45:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL46:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL47:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL48:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL01:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL02:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL03:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL04:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL05:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL06:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL07:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL08:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL09:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL10:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL11:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL12:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL13:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL14:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL15:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL16:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL17:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL18:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL19:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL20:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL21:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL22:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL23:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL24:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL25:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL26:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL27:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL28:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL29:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL30:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL31:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL32:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL01:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL02:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL03:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL04:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL05:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL06:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL07:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL08:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL09:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL10:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL11:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL12:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL13:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL14:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL15:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL16:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL17:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL18:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL19:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL20:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL21:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL22:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL23:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL24:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL25:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL26:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL27:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL28:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL29:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL30:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL31:ROT + - \D3D::TOP.IONS.CER.CERAUTO.VERTICAL.CHANNEL32:ROT + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:TIMEBASE + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:TIMEBASE + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONRATE1 + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONRATE3 + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONRATE4 + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONSRATE + - \D3D::TOP.MSE.ANALYSIS_01:MSEP01 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP02 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP03 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP04 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP05 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP06 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP07 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP08 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP09 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP10 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP11 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP12 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP13 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP14 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP15 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP16 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP17 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP18 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP19 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP20 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP21 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP22 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP23 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP24 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP25 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP26 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP27 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP28 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP29 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP30 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP31 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP32 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP33 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP34 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP35 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP36 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP37 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP38 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP39 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP40 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP41 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP42 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP43 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP44 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP45 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP46 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP47 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP48 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP49 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP50 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP51 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP52 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP53 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP54 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP55 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP56 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP57 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP58 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP59 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP60 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP61 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP62 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP63 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP64 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP65 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP66 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP67 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP68 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP69 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP01:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP02:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP03:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP04:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP05:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP06:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP07:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP08:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP09:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP10:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP11:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP12:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP13:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP14:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP15:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP16:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP17:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP18:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP19:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP20:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP21:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP22:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP23:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP24:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP25:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP26:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP27:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP28:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP29:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP30:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP31:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP32:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP33:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP34:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP35:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP36:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP37:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP38:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP39:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP40:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP41:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP42:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP43:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP44:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP45:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP46:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP47:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP48:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP49:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP50:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP51:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP52:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP53:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP54:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP55:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP56:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP57:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP58:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP59:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP60:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP61:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP62:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP63:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP64:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP65:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP66:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP67:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP68:TIME + - \D3D::TOP.MSE.ANALYSIS_01:MSEP69:TIME + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L01_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L02_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L03_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L04_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L05_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L06_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L07_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L08_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L09_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L10_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L11_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L12_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L13_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L14_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L15_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L16_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L17_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L18_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L19_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L20_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L21_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L22_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L23_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L24_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U01_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U02_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U03_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U04_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U05_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U06_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U07_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U08_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U09_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U10_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U11_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U12_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U13_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U14_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U15_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U16_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U17_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U18_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U19_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U20_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U21_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U22_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U23_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U24_V + - \D3D::TOP.ELECTRONS.BCI.DPD.R0:DENUF + - \D3D::TOP.ELECTRONS.BCI.DPD.V1:DENUF + - \D3D::TOP.ELECTRONS.BCI.DPD.V2:DENUF + - \D3D::TOP.ELECTRONS.BCI.DPD.V3:DENUF + - \D3D::TOP.SPECTROSCOPY.VB.TUBE01 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE02 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE03 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE04 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE05 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE06 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE07 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE08 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE09 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE10 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE11 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE12 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE13 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE14 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE15 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE16 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE17 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE18 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE19 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE20 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE21 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE22 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE23 + - \D3D::TOP.SPECTROSCOPY.VB.TUBE24 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD01 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD02 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD03 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD04 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD05 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD06 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD07 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD08 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD09 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD10 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD11 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD12 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD13 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD14 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD15 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD16 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD17 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD18 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD19 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD20 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD21 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD22 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD23 + - \D3D::TOP.SPECTROSCOPY.VB.CHORD24 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_01 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_02 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_03 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_04 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_05 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_06 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_07 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_08 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_09 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_10 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_11 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_12 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_13 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_14 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_15 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_16 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_17 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_18 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_19 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_20 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_21 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_22 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_23 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_24 + - \SPECTROSCOPY::FS01 + - \SPECTROSCOPY::FS02 + - \SPECTROSCOPY::FS03 + - \SPECTROSCOPY::FS04 + - \SPECTROSCOPY::FS05 + - \SPECTROSCOPY::FS06 + - \SPECTROSCOPY::FS07 + - \SPECTROSCOPY::FS08 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT01 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT02 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT03 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT04 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT05 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT06 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT07 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT08 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT09 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT10 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT11 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT12 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT13 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT14 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT15 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT16 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT17 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT18 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT19 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT20 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT21 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT22 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT23 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT24 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT25 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT26 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT27 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT28 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT29 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT30 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT31 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT32 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT33 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT34 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT35 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT36 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT37 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT38 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT39 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT40 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT41 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT42 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT43 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT44 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT45 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT46 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT47 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT48 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT49 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT50 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT51 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT52 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT53 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT54 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT55 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT56 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT57 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT58 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT59 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT60 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT61 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT62 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT63 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT64 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT65 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT66 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT67 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT68 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT69 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT70 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT71 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT72 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT73 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT74 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT75 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT76 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT77 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT78 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT79 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT80 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT81 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT82 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT83 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT84 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT85 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT86 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT87 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT88 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT89 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT90 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT91 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT92 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT93 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT94 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT95 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT96 + - \D3D::TOP.NB.NB15L.BEAMTARGET + - \D3D::TOP.NB.NB15L.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB15L.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB15L.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB15L.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB15L.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB15L.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB15L.BEAMTARGET:DUR + - \D3D::TOP.NB.NB15L.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB15L.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB15L.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB15L.BEAMTARGET:START + - \D3D::TOP.NB.NB15L.BEAMTARGET:STATE + - \D3D::TOP.NB.NB15L.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB15L.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB15L.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB15L.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB15L.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB15L.OANB + - \D3D::TOP.NB.NB15L.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB15L.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB15L.OANB:CYLLTH_F + - \D3D::TOP.NB.NB15L.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB15L.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB15L.OANB:SRCGAP_L + - \D3D::TOP.NB.NB15L.OANB:SRCGAP_R + - \D3D::TOP.NB.NB15L.OANB:SRCPTCH + - \D3D::TOP.NB.NB15L:BEAMSTAT + - \D3D::TOP.NB.NB15L:BEAMSTATF + - \D3D::TOP.NB.NB15L:CURRENT + - \D3D::TOP.NB.NB15L:ETA1 + - \D3D::TOP.NB.NB15L:F0 + - \D3D::TOP.NB.NB15L:FIRED + - \D3D::TOP.NB.NB15L:FIRED:INFO + - \D3D::TOP.NB.NB15L:GAS + - \D3D::TOP.NB.NB15L:HEADPOINT + - \D3D::TOP.NB.NB15L:IFIX + - \D3D::TOP.NB.NB15L:INTERCEPT + - \D3D::TOP.NB.NB15L:IONSRCTYPE + - \D3D::TOP.NB.NB15L:MODE + - \D3D::TOP.NB.NB15L:MODULATION + - \D3D::TOP.NB.NB15L:NBSHOT + - \D3D::TOP.NB.NB15L:NBVAC_SCALAR + - \D3D::TOP.NB.NB15L:NEUT_GAS_FL + - \D3D::TOP.NB.NB15L:PABS + - \D3D::TOP.NB.NB15L:PERVEANCE + - \D3D::TOP.NB.NB15L:PINJF_15L + - \D3D::TOP.NB.NB15L:PINJ_15L + - \D3D::TOP.NB.NB15L:PINJ_SCALAR + - \D3D::TOP.NB.NB15L:POWER + - \D3D::TOP.NB.NB15L:PSHINE + - \D3D::TOP.NB.NB15L:PTDATA_CAL + - \D3D::TOP.NB.NB15L:PTDATA_CALF + - \D3D::TOP.NB.NB15L:PTDATA_RAW + - \D3D::TOP.NB.NB15L:PTDATA_RAWF + - \D3D::TOP.NB.NB15L:REAL32 + - \D3D::TOP.NB.NB15L:SHINE_THRU + - \D3D::TOP.NB.NB15L:SING_SRC_FAC + - \D3D::TOP.NB.NB15L:SLOPE + - \D3D::TOP.NB.NB15L:SYNC + - \D3D::TOP.NB.NB15L:TE + - \D3D::TOP.NB.NB15L:THRESHOLD + - \D3D::TOP.NB.NB15L:THRESHOLD:INFO + - \D3D::TOP.NB.NB15L:TINJ_15L + - \D3D::TOP.NB.NB15L:VBEAM + - \D3D::TOP.NB.NB15L:VBEAMF + - \D3D::TOP.NB.NB15L:VOLTAGE + - \D3D::TOP.NB.NB15L:VOLTAGE_CAL + - \D3D::TOP.NB.NB15L:VOLTAGE_CALF + - \D3D::TOP.NB.NB15R.BEAMTARGET + - \D3D::TOP.NB.NB15R.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB15R.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB15R.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB15R.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB15R.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB15R.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB15R.BEAMTARGET:DUR + - \D3D::TOP.NB.NB15R.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB15R.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB15R.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB15R.BEAMTARGET:START + - \D3D::TOP.NB.NB15R.BEAMTARGET:STATE + - \D3D::TOP.NB.NB15R.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB15R.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB15R.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB15R.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB15R.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB15R.OANB + - \D3D::TOP.NB.NB15R.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB15R.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB15R.OANB:CYLLTH_F + - \D3D::TOP.NB.NB15R.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB15R.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB15R.OANB:SRCGAP_L + - \D3D::TOP.NB.NB15R.OANB:SRCGAP_R + - \D3D::TOP.NB.NB15R.OANB:SRCPTCH + - \D3D::TOP.NB.NB15R:BEAMSTAT + - \D3D::TOP.NB.NB15R:BEAMSTATF + - \D3D::TOP.NB.NB15R:CURRENT + - \D3D::TOP.NB.NB15R:ETA1 + - \D3D::TOP.NB.NB15R:F0 + - \D3D::TOP.NB.NB15R:FIRED + - \D3D::TOP.NB.NB15R:FIRED:INFO + - \D3D::TOP.NB.NB15R:GAS + - \D3D::TOP.NB.NB15R:HEADPOINT + - \D3D::TOP.NB.NB15R:IFIX + - \D3D::TOP.NB.NB15R:INTERCEPT + - \D3D::TOP.NB.NB15R:IONSRCTYPE + - \D3D::TOP.NB.NB15R:MODE + - \D3D::TOP.NB.NB15R:MODULATION + - \D3D::TOP.NB.NB15R:NBSHOT + - \D3D::TOP.NB.NB15R:NBVAC_SCALAR + - \D3D::TOP.NB.NB15R:NEUT_GAS_FL + - \D3D::TOP.NB.NB15R:PABS + - \D3D::TOP.NB.NB15R:PERVEANCE + - \D3D::TOP.NB.NB15R:PINJF_15R + - \D3D::TOP.NB.NB15R:PINJ_15R + - \D3D::TOP.NB.NB15R:PINJ_SCALAR + - \D3D::TOP.NB.NB15R:POWER + - \D3D::TOP.NB.NB15R:PSHINE + - \D3D::TOP.NB.NB15R:PTDATA_CAL + - \D3D::TOP.NB.NB15R:PTDATA_CALF + - \D3D::TOP.NB.NB15R:PTDATA_RAW + - \D3D::TOP.NB.NB15R:PTDATA_RAWF + - \D3D::TOP.NB.NB15R:REAL32 + - \D3D::TOP.NB.NB15R:SHINE_THRU + - \D3D::TOP.NB.NB15R:SING_SRC_FAC + - \D3D::TOP.NB.NB15R:SLOPE + - \D3D::TOP.NB.NB15R:SYNC + - \D3D::TOP.NB.NB15R:TE + - \D3D::TOP.NB.NB15R:THRESHOLD + - \D3D::TOP.NB.NB15R:THRESHOLD:INFO + - \D3D::TOP.NB.NB15R:TINJ_15R + - \D3D::TOP.NB.NB15R:VBEAM + - \D3D::TOP.NB.NB15R:VBEAMF + - \D3D::TOP.NB.NB15R:VOLTAGE + - \D3D::TOP.NB.NB15R:VOLTAGE_CAL + - \D3D::TOP.NB.NB15R:VOLTAGE_CALF + - \D3D::TOP.NB.NB21L.BEAMTARGET + - \D3D::TOP.NB.NB21L.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB21L.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB21L.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB21L.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB21L.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB21L.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB21L.BEAMTARGET:DUR + - \D3D::TOP.NB.NB21L.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB21L.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB21L.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB21L.BEAMTARGET:START + - \D3D::TOP.NB.NB21L.BEAMTARGET:STATE + - \D3D::TOP.NB.NB21L.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB21L.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB21L.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB21L.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB21L.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB21L.OANB + - \D3D::TOP.NB.NB21L.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB21L.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB21L.OANB:CYLLTH_F + - \D3D::TOP.NB.NB21L.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB21L.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB21L.OANB:SRCGAP_L + - \D3D::TOP.NB.NB21L.OANB:SRCGAP_R + - \D3D::TOP.NB.NB21L.OANB:SRCPTCH + - \D3D::TOP.NB.NB21L:BEAMSTAT + - \D3D::TOP.NB.NB21L:BEAMSTATF + - \D3D::TOP.NB.NB21L:CURRENT + - \D3D::TOP.NB.NB21L:ETA1 + - \D3D::TOP.NB.NB21L:F0 + - \D3D::TOP.NB.NB21L:FIRED + - \D3D::TOP.NB.NB21L:FIRED:INFO + - \D3D::TOP.NB.NB21L:GAS + - \D3D::TOP.NB.NB21L:HEADPOINT + - \D3D::TOP.NB.NB21L:IFIX + - \D3D::TOP.NB.NB21L:INTERCEPT + - \D3D::TOP.NB.NB21L:IONSRCTYPE + - \D3D::TOP.NB.NB21L:MODE + - \D3D::TOP.NB.NB21L:MODULATION + - \D3D::TOP.NB.NB21L:NBSHOT + - \D3D::TOP.NB.NB21L:NBVAC_SCALAR + - \D3D::TOP.NB.NB21L:NEUT_GAS_FL + - \D3D::TOP.NB.NB21L:PABS + - \D3D::TOP.NB.NB21L:PERVEANCE + - \D3D::TOP.NB.NB21L:PINJF_21L + - \D3D::TOP.NB.NB21L:PINJ_21L + - \D3D::TOP.NB.NB21L:PINJ_SCALAR + - \D3D::TOP.NB.NB21L:POWER + - \D3D::TOP.NB.NB21L:PSHINE + - \D3D::TOP.NB.NB21L:PTDATA_CAL + - \D3D::TOP.NB.NB21L:PTDATA_CALF + - \D3D::TOP.NB.NB21L:PTDATA_RAW + - \D3D::TOP.NB.NB21L:PTDATA_RAWF + - \D3D::TOP.NB.NB21L:REAL32 + - \D3D::TOP.NB.NB21L:SHINE_THRU + - \D3D::TOP.NB.NB21L:SING_SRC_FAC + - \D3D::TOP.NB.NB21L:SLOPE + - \D3D::TOP.NB.NB21L:SYNC + - \D3D::TOP.NB.NB21L:TE + - \D3D::TOP.NB.NB21L:THRESHOLD + - \D3D::TOP.NB.NB21L:THRESHOLD:INFO + - \D3D::TOP.NB.NB21L:TINJ_21L + - \D3D::TOP.NB.NB21L:VBEAM + - \D3D::TOP.NB.NB21L:VBEAMF + - \D3D::TOP.NB.NB21L:VOLTAGE + - \D3D::TOP.NB.NB21L:VOLTAGE_CAL + - \D3D::TOP.NB.NB21L:VOLTAGE_CALF + - \D3D::TOP.NB.NB21R.BEAMTARGET + - \D3D::TOP.NB.NB21R.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB21R.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB21R.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB21R.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB21R.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB21R.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB21R.BEAMTARGET:DUR + - \D3D::TOP.NB.NB21R.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB21R.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB21R.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB21R.BEAMTARGET:START + - \D3D::TOP.NB.NB21R.BEAMTARGET:STATE + - \D3D::TOP.NB.NB21R.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB21R.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB21R.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB21R.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB21R.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB21R.OANB + - \D3D::TOP.NB.NB21R.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB21R.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB21R.OANB:CYLLTH_F + - \D3D::TOP.NB.NB21R.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB21R.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB21R.OANB:SRCGAP_L + - \D3D::TOP.NB.NB21R.OANB:SRCGAP_R + - \D3D::TOP.NB.NB21R.OANB:SRCPTCH + - \D3D::TOP.NB.NB21R:BEAMSTAT + - \D3D::TOP.NB.NB21R:BEAMSTATF + - \D3D::TOP.NB.NB21R:CURRENT + - \D3D::TOP.NB.NB21R:ETA1 + - \D3D::TOP.NB.NB21R:F0 + - \D3D::TOP.NB.NB21R:FIRED + - \D3D::TOP.NB.NB21R:FIRED:INFO + - \D3D::TOP.NB.NB21R:GAS + - \D3D::TOP.NB.NB21R:HEADPOINT + - \D3D::TOP.NB.NB21R:IFIX + - \D3D::TOP.NB.NB21R:INTERCEPT + - \D3D::TOP.NB.NB21R:IONSRCTYPE + - \D3D::TOP.NB.NB21R:MODE + - \D3D::TOP.NB.NB21R:MODULATION + - \D3D::TOP.NB.NB21R:NBSHOT + - \D3D::TOP.NB.NB21R:NBVAC_SCALAR + - \D3D::TOP.NB.NB21R:NEUT_GAS_FL + - \D3D::TOP.NB.NB21R:PABS + - \D3D::TOP.NB.NB21R:PERVEANCE + - \D3D::TOP.NB.NB21R:PINJF_21R + - \D3D::TOP.NB.NB21R:PINJ_21R + - \D3D::TOP.NB.NB21R:PINJ_SCALAR + - \D3D::TOP.NB.NB21R:POWER + - \D3D::TOP.NB.NB21R:PSHINE + - \D3D::TOP.NB.NB21R:PTDATA_CAL + - \D3D::TOP.NB.NB21R:PTDATA_CALF + - \D3D::TOP.NB.NB21R:PTDATA_RAW + - \D3D::TOP.NB.NB21R:PTDATA_RAWF + - \D3D::TOP.NB.NB21R:REAL32 + - \D3D::TOP.NB.NB21R:SHINE_THRU + - \D3D::TOP.NB.NB21R:SING_SRC_FAC + - \D3D::TOP.NB.NB21R:SLOPE + - \D3D::TOP.NB.NB21R:SYNC + - \D3D::TOP.NB.NB21R:TE + - \D3D::TOP.NB.NB21R:THRESHOLD + - \D3D::TOP.NB.NB21R:THRESHOLD:INFO + - \D3D::TOP.NB.NB21R:TINJ_21R + - \D3D::TOP.NB.NB21R:VBEAM + - \D3D::TOP.NB.NB21R:VBEAMF + - \D3D::TOP.NB.NB21R:VOLTAGE + - \D3D::TOP.NB.NB21R:VOLTAGE_CAL + - \D3D::TOP.NB.NB21R:VOLTAGE_CALF + - \D3D::TOP.NB.NB30L.BEAMTARGET + - \D3D::TOP.NB.NB30L.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB30L.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB30L.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB30L.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB30L.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB30L.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB30L.BEAMTARGET:DUR + - \D3D::TOP.NB.NB30L.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB30L.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB30L.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB30L.BEAMTARGET:START + - \D3D::TOP.NB.NB30L.BEAMTARGET:STATE + - \D3D::TOP.NB.NB30L.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB30L.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB30L.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB30L.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB30L.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB30L.OANB + - \D3D::TOP.NB.NB30L.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB30L.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB30L.OANB:CYLLTH_F + - \D3D::TOP.NB.NB30L.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB30L.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB30L.OANB:SRCGAP_L + - \D3D::TOP.NB.NB30L.OANB:SRCGAP_R + - \D3D::TOP.NB.NB30L.OANB:SRCPTCH + - \D3D::TOP.NB.NB30L:BEAMSTAT + - \D3D::TOP.NB.NB30L:BEAMSTATF + - \D3D::TOP.NB.NB30L:CURRENT + - \D3D::TOP.NB.NB30L:ETA1 + - \D3D::TOP.NB.NB30L:F0 + - \D3D::TOP.NB.NB30L:FIRED + - \D3D::TOP.NB.NB30L:FIRED:INFO + - \D3D::TOP.NB.NB30L:GAS + - \D3D::TOP.NB.NB30L:HEADPOINT + - \D3D::TOP.NB.NB30L:IFIX + - \D3D::TOP.NB.NB30L:INTERCEPT + - \D3D::TOP.NB.NB30L:IONSRCTYPE + - \D3D::TOP.NB.NB30L:MODE + - \D3D::TOP.NB.NB30L:MODULATION + - \D3D::TOP.NB.NB30L:NBSHOT + - \D3D::TOP.NB.NB30L:NBVAC_SCALAR + - \D3D::TOP.NB.NB30L:NEUT_GAS_FL + - \D3D::TOP.NB.NB30L:PABS + - \D3D::TOP.NB.NB30L:PERVEANCE + - \D3D::TOP.NB.NB30L:PINJF_30L + - \D3D::TOP.NB.NB30L:PINJ_30L + - \D3D::TOP.NB.NB30L:PINJ_SCALAR + - \D3D::TOP.NB.NB30L:POWER + - \D3D::TOP.NB.NB30L:PSHINE + - \D3D::TOP.NB.NB30L:PTDATA_CAL + - \D3D::TOP.NB.NB30L:PTDATA_CALF + - \D3D::TOP.NB.NB30L:PTDATA_RAW + - \D3D::TOP.NB.NB30L:PTDATA_RAWF + - \D3D::TOP.NB.NB30L:REAL32 + - \D3D::TOP.NB.NB30L:SHINE_THRU + - \D3D::TOP.NB.NB30L:SING_SRC_FAC + - \D3D::TOP.NB.NB30L:SLOPE + - \D3D::TOP.NB.NB30L:SYNC + - \D3D::TOP.NB.NB30L:TE + - \D3D::TOP.NB.NB30L:THRESHOLD + - \D3D::TOP.NB.NB30L:THRESHOLD:INFO + - \D3D::TOP.NB.NB30L:TINJ_30L + - \D3D::TOP.NB.NB30L:VBEAM + - \D3D::TOP.NB.NB30L:VBEAMF + - \D3D::TOP.NB.NB30L:VOLTAGE + - \D3D::TOP.NB.NB30L:VOLTAGE_CAL + - \D3D::TOP.NB.NB30L:VOLTAGE_CALF + - \D3D::TOP.NB.NB30R.BEAMTARGET + - \D3D::TOP.NB.NB30R.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB30R.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB30R.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB30R.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB30R.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB30R.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB30R.BEAMTARGET:DUR + - \D3D::TOP.NB.NB30R.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB30R.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB30R.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB30R.BEAMTARGET:START + - \D3D::TOP.NB.NB30R.BEAMTARGET:STATE + - \D3D::TOP.NB.NB30R.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB30R.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB30R.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB30R.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB30R.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB30R.OANB + - \D3D::TOP.NB.NB30R.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB30R.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB30R.OANB:CYLLTH_F + - \D3D::TOP.NB.NB30R.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB30R.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB30R.OANB:SRCGAP_L + - \D3D::TOP.NB.NB30R.OANB:SRCGAP_R + - \D3D::TOP.NB.NB30R.OANB:SRCPTCH + - \D3D::TOP.NB.NB30R:BEAMSTAT + - \D3D::TOP.NB.NB30R:BEAMSTATF + - \D3D::TOP.NB.NB30R:CURRENT + - \D3D::TOP.NB.NB30R:ETA1 + - \D3D::TOP.NB.NB30R:F0 + - \D3D::TOP.NB.NB30R:FIRED + - \D3D::TOP.NB.NB30R:FIRED:INFO + - \D3D::TOP.NB.NB30R:GAS + - \D3D::TOP.NB.NB30R:HEADPOINT + - \D3D::TOP.NB.NB30R:IFIX + - \D3D::TOP.NB.NB30R:INTERCEPT + - \D3D::TOP.NB.NB30R:IONSRCTYPE + - \D3D::TOP.NB.NB30R:MODE + - \D3D::TOP.NB.NB30R:MODULATION + - \D3D::TOP.NB.NB30R:NBSHOT + - \D3D::TOP.NB.NB30R:NBVAC_SCALAR + - \D3D::TOP.NB.NB30R:NEUT_GAS_FL + - \D3D::TOP.NB.NB30R:PABS + - \D3D::TOP.NB.NB30R:PERVEANCE + - \D3D::TOP.NB.NB30R:PINJF_30R + - \D3D::TOP.NB.NB30R:PINJ_30R + - \D3D::TOP.NB.NB30R:PINJ_SCALAR + - \D3D::TOP.NB.NB30R:POWER + - \D3D::TOP.NB.NB30R:PSHINE + - \D3D::TOP.NB.NB30R:PTDATA_CAL + - \D3D::TOP.NB.NB30R:PTDATA_CALF + - \D3D::TOP.NB.NB30R:PTDATA_RAW + - \D3D::TOP.NB.NB30R:PTDATA_RAWF + - \D3D::TOP.NB.NB30R:REAL32 + - \D3D::TOP.NB.NB30R:SHINE_THRU + - \D3D::TOP.NB.NB30R:SING_SRC_FAC + - \D3D::TOP.NB.NB30R:SLOPE + - \D3D::TOP.NB.NB30R:SYNC + - \D3D::TOP.NB.NB30R:TE + - \D3D::TOP.NB.NB30R:THRESHOLD + - \D3D::TOP.NB.NB30R:THRESHOLD:INFO + - \D3D::TOP.NB.NB30R:TINJ_30R + - \D3D::TOP.NB.NB30R:VBEAM + - \D3D::TOP.NB.NB30R:VBEAMF + - \D3D::TOP.NB.NB30R:VOLTAGE + - \D3D::TOP.NB.NB30R:VOLTAGE_CAL + - \D3D::TOP.NB.NB30R:VOLTAGE_CALF + - \D3D::TOP.NB.NB33L.BEAMTARGET + - \D3D::TOP.NB.NB33L.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB33L.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB33L.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB33L.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB33L.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB33L.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB33L.BEAMTARGET:DUR + - \D3D::TOP.NB.NB33L.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB33L.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB33L.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB33L.BEAMTARGET:START + - \D3D::TOP.NB.NB33L.BEAMTARGET:STATE + - \D3D::TOP.NB.NB33L.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB33L.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB33L.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB33L.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB33L.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB33L.OANB + - \D3D::TOP.NB.NB33L.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB33L.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB33L.OANB:CYLLTH_F + - \D3D::TOP.NB.NB33L.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB33L.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB33L.OANB:SRCGAP_L + - \D3D::TOP.NB.NB33L.OANB:SRCGAP_R + - \D3D::TOP.NB.NB33L.OANB:SRCPTCH + - \D3D::TOP.NB.NB33L:BEAMSTAT + - \D3D::TOP.NB.NB33L:BEAMSTATF + - \D3D::TOP.NB.NB33L:CURRENT + - \D3D::TOP.NB.NB33L:ETA1 + - \D3D::TOP.NB.NB33L:F0 + - \D3D::TOP.NB.NB33L:FIRED + - \D3D::TOP.NB.NB33L:FIRED:INFO + - \D3D::TOP.NB.NB33L:GAS + - \D3D::TOP.NB.NB33L:HEADPOINT + - \D3D::TOP.NB.NB33L:IFIX + - \D3D::TOP.NB.NB33L:INTERCEPT + - \D3D::TOP.NB.NB33L:IONSRCTYPE + - \D3D::TOP.NB.NB33L:MODE + - \D3D::TOP.NB.NB33L:MODULATION + - \D3D::TOP.NB.NB33L:NBSHOT + - \D3D::TOP.NB.NB33L:NBVAC_SCALAR + - \D3D::TOP.NB.NB33L:NEUT_GAS_FL + - \D3D::TOP.NB.NB33L:PABS + - \D3D::TOP.NB.NB33L:PERVEANCE + - \D3D::TOP.NB.NB33L:PINJF_33L + - \D3D::TOP.NB.NB33L:PINJ_33L + - \D3D::TOP.NB.NB33L:PINJ_SCALAR + - \D3D::TOP.NB.NB33L:POWER + - \D3D::TOP.NB.NB33L:PSHINE + - \D3D::TOP.NB.NB33L:PTDATA_CAL + - \D3D::TOP.NB.NB33L:PTDATA_CALF + - \D3D::TOP.NB.NB33L:PTDATA_RAW + - \D3D::TOP.NB.NB33L:PTDATA_RAWF + - \D3D::TOP.NB.NB33L:REAL32 + - \D3D::TOP.NB.NB33L:SHINE_THRU + - \D3D::TOP.NB.NB33L:SING_SRC_FAC + - \D3D::TOP.NB.NB33L:SLOPE + - \D3D::TOP.NB.NB33L:SYNC + - \D3D::TOP.NB.NB33L:TE + - \D3D::TOP.NB.NB33L:THRESHOLD + - \D3D::TOP.NB.NB33L:THRESHOLD:INFO + - \D3D::TOP.NB.NB33L:TINJ_33L + - \D3D::TOP.NB.NB33L:VBEAM + - \D3D::TOP.NB.NB33L:VBEAMF + - \D3D::TOP.NB.NB33L:VOLTAGE + - \D3D::TOP.NB.NB33L:VOLTAGE_CAL + - \D3D::TOP.NB.NB33L:VOLTAGE_CALF + - \D3D::TOP.NB.NB33R.BEAMTARGET + - \D3D::TOP.NB.NB33R.BEAMTARGET:ACTUAL + - \D3D::TOP.NB.NB33R.BEAMTARGET:ACTUALF + - \D3D::TOP.NB.NB33R.BEAMTARGET:BEAMCOM + - \D3D::TOP.NB.NB33R.BEAMTARGET:BEAMPROG + - \D3D::TOP.NB.NB33R.BEAMTARGET:DIFF + - \D3D::TOP.NB.NB33R.BEAMTARGET:DIFFF + - \D3D::TOP.NB.NB33R.BEAMTARGET:DUR + - \D3D::TOP.NB.NB33R.BEAMTARGET:OFFTRAIN + - \D3D::TOP.NB.NB33R.BEAMTARGET:ONTRAIN + - \D3D::TOP.NB.NB33R.BEAMTARGET:PHASE + - \D3D::TOP.NB.NB33R.BEAMTARGET:START + - \D3D::TOP.NB.NB33R.BEAMTARGET:STATE + - \D3D::TOP.NB.NB33R.BEAMTARGET:TARGET + - \D3D::TOP.NB.NB33R.BEAMTARGET:TARGETF + - \D3D::TOP.NB.NB33R.BEAMTARGET:TDAT + - \D3D::TOP.NB.NB33R.BEAMTARGET:TDATF + - \D3D::TOP.NB.NB33R.BEAMTARGET:TIMEBASE + - \D3D::TOP.NB.NB33R.OANB + - \D3D::TOP.NB.NB33R.OANB:BLPTCH_CAD + - \D3D::TOP.NB.NB33R.OANB:BLPTCH_INCL + - \D3D::TOP.NB.NB33R.OANB:CYLLTH_F + - \D3D::TOP.NB.NB33R.OANB:CYLLTH_LR + - \D3D::TOP.NB.NB33R.OANB:CYLLTH_RR + - \D3D::TOP.NB.NB33R.OANB:SRCGAP_L + - \D3D::TOP.NB.NB33R.OANB:SRCGAP_R + - \D3D::TOP.NB.NB33R.OANB:SRCPTCH + - \D3D::TOP.NB.NB33R:BEAMSTAT + - \D3D::TOP.NB.NB33R:BEAMSTATF + - \D3D::TOP.NB.NB33R:CURRENT + - \D3D::TOP.NB.NB33R:ETA1 + - \D3D::TOP.NB.NB33R:F0 + - \D3D::TOP.NB.NB33R:FIRED + - \D3D::TOP.NB.NB33R:FIRED:INFO + - \D3D::TOP.NB.NB33R:GAS + - \D3D::TOP.NB.NB33R:HEADPOINT + - \D3D::TOP.NB.NB33R:IFIX + - \D3D::TOP.NB.NB33R:INTERCEPT + - \D3D::TOP.NB.NB33R:IONSRCTYPE + - \D3D::TOP.NB.NB33R:MODE + - \D3D::TOP.NB.NB33R:MODULATION + - \D3D::TOP.NB.NB33R:NBSHOT + - \D3D::TOP.NB.NB33R:NBVAC_SCALAR + - \D3D::TOP.NB.NB33R:NEUT_GAS_FL + - \D3D::TOP.NB.NB33R:PABS + - \D3D::TOP.NB.NB33R:PERVEANCE + - \D3D::TOP.NB.NB33R:PINJF_33R + - \D3D::TOP.NB.NB33R:PINJ_33R + - \D3D::TOP.NB.NB33R:PINJ_SCALAR + - \D3D::TOP.NB.NB33R:POWER + - \D3D::TOP.NB.NB33R:PSHINE + - \D3D::TOP.NB.NB33R:PTDATA_CAL + - \D3D::TOP.NB.NB33R:PTDATA_CALF + - \D3D::TOP.NB.NB33R:PTDATA_RAW + - \D3D::TOP.NB.NB33R:PTDATA_RAWF + - \D3D::TOP.NB.NB33R:REAL32 + - \D3D::TOP.NB.NB33R:SHINE_THRU + - \D3D::TOP.NB.NB33R:SING_SRC_FAC + - \D3D::TOP.NB.NB33R:SLOPE + - \D3D::TOP.NB.NB33R:SYNC + - \D3D::TOP.NB.NB33R:TE + - \D3D::TOP.NB.NB33R:THRESHOLD + - \D3D::TOP.NB.NB33R:THRESHOLD:INFO + - \D3D::TOP.NB.NB33R:TINJ_33R + - \D3D::TOP.NB.NB33R:VBEAM + - \D3D::TOP.NB.NB33R:VBEAMF + - \D3D::TOP.NB.NB33R:VOLTAGE + - \D3D::TOP.NB.NB33R:VOLTAGE_CAL + - \D3D::TOP.NB.NB33R:VOLTAGE_CALF + - \D3D::TOP.RF.ECH.BORIS:ECBORAZIANG + - \D3D::TOP.RF.ECH.BORIS:ECBORFPWRC + - \D3D::TOP.RF.ECH.BORIS:ECBORPOLANG + - \D3D::TOP.RF.ECH.BORIS:ECBORPOLCNT + - \D3D::TOP.RF.ECH.BORIS:ECBORSTAT + - \D3D::TOP.RF.ECH.BORIS:ECBORTORCNT + - \D3D::TOP.RF.ECH.BORIS:ECBORXMFRAC + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHEAZIANG + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHEFPWRC + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHEPOLANG + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHEPOLCNT + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHESTAT + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHETORCNT + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHEXMFRAC + - \D3D::TOP.RF.ECH.DOROTHY:ECDORAZIANG + - \D3D::TOP.RF.ECH.DOROTHY:ECDORFPWRC + - \D3D::TOP.RF.ECH.DOROTHY:ECDORPOLANG + - \D3D::TOP.RF.ECH.DOROTHY:ECDORPOLCNT + - \D3D::TOP.RF.ECH.DOROTHY:ECDORSTAT + - \D3D::TOP.RF.ECH.DOROTHY:ECDORTORCNT + - \D3D::TOP.RF.ECH.DOROTHY:ECDORXMFRAC + - \D3D::TOP.RF.ECH.HAN:ECHANAZIANG + - \D3D::TOP.RF.ECH.HAN:ECHANDLPWRC + - \D3D::TOP.RF.ECH.HAN:ECHANPOLANG + - \D3D::TOP.RF.ECH.HAN:ECHANPOLCNT + - \D3D::TOP.RF.ECH.HAN:ECHANSTAT + - \D3D::TOP.RF.ECH.HAN:ECHANTORCNT + - \D3D::TOP.RF.ECH.HAN:ECHANXMFRAC + - \D3D::TOP.RF.ECH.KATYA:ECKATAZIANG + - \D3D::TOP.RF.ECH.KATYA:ECKATFPWRC + - \D3D::TOP.RF.ECH.KATYA:ECKATPOLANG + - \D3D::TOP.RF.ECH.KATYA:ECKATPOLCNT + - \D3D::TOP.RF.ECH.KATYA:ECKATSTAT + - \D3D::TOP.RF.ECH.KATYA:ECKATTORCNT + - \D3D::TOP.RF.ECH.KATYA:ECKATXMFRAC + - \D3D::TOP.RF.ECH.LEIA:ECLEIAZIANG + - \D3D::TOP.RF.ECH.LEIA:ECLEIFPWRC + - \D3D::TOP.RF.ECH.LEIA:ECLEIPOLANG + - \D3D::TOP.RF.ECH.LEIA:ECLEIPOLCNT + - \D3D::TOP.RF.ECH.LEIA:ECLEISTAT + - \D3D::TOP.RF.ECH.LEIA:ECLEITORCNT + - \D3D::TOP.RF.ECH.LEIA:ECLEIXMFRAC + - \D3D::TOP.RF.ECH.LION:ECLIOAZIANG + - \D3D::TOP.RF.ECH.LION:ECLIOFPWRC + - \D3D::TOP.RF.ECH.LION:ECLIOPOLANG + - \D3D::TOP.RF.ECH.LION:ECLIOPOLCNT + - \D3D::TOP.RF.ECH.LION:ECLIOSTAT + - \D3D::TOP.RF.ECH.LION:ECLIOTORCNT + - \D3D::TOP.RF.ECH.LION:ECLIOXMFRAC + - \D3D::TOP.RF.ECH.LUKE:ECLUKAZIANG + - \D3D::TOP.RF.ECH.LUKE:ECLUKFPWRC + - \D3D::TOP.RF.ECH.LUKE:ECLUKPOLANG + - \D3D::TOP.RF.ECH.LUKE:ECLUKPOLCNT + - \D3D::TOP.RF.ECH.LUKE:ECLUKSTAT + - \D3D::TOP.RF.ECH.LUKE:ECLUKTORCNT + - \D3D::TOP.RF.ECH.LUKE:ECLUKXMFRAC + - \D3D::TOP.RF.ECH.NASA:ECNASAZIANG + - \D3D::TOP.RF.ECH.NASA:ECNASFPWRC + - \D3D::TOP.RF.ECH.NASA:ECNASPOLANG + - \D3D::TOP.RF.ECH.NASA:ECNASPOLCNT + - \D3D::TOP.RF.ECH.NASA:ECNASSTAT + - \D3D::TOP.RF.ECH.NASA:ECNASTORCNT + - \D3D::TOP.RF.ECH.NASA:ECNASXMFRAC + - \D3D::TOP.RF.ECH.NATASHA:ECNATAZIANG + - \D3D::TOP.RF.ECH.NATASHA:ECNATFPWRC + - \D3D::TOP.RF.ECH.NATASHA:ECNATPOLANG + - \D3D::TOP.RF.ECH.NATASHA:ECNATPOLCNT + - \D3D::TOP.RF.ECH.NATASHA:ECNATSTAT + - \D3D::TOP.RF.ECH.NATASHA:ECNATTORCNT + - \D3D::TOP.RF.ECH.NATASHA:ECNATXMFRAC + - \D3D::TOP.RF.ECH.R2D2:ECR2DAZIANG + - \D3D::TOP.RF.ECH.R2D2:ECR2DFPWRC + - \D3D::TOP.RF.ECH.R2D2:ECR2DPOLANG + - \D3D::TOP.RF.ECH.R2D2:ECR2DPOLCNT + - \D3D::TOP.RF.ECH.R2D2:ECR2DSTAT + - \D3D::TOP.RF.ECH.R2D2:ECR2DTORCNT + - \D3D::TOP.RF.ECH.R2D2:ECR2DXMFRAC + - \D3D::TOP.RF.ECH.SCARECROW:ECSCAAZIANG + - \D3D::TOP.RF.ECH.SCARECROW:ECSCAFPWRC + - \D3D::TOP.RF.ECH.SCARECROW:ECSCAPOLANG + - \D3D::TOP.RF.ECH.SCARECROW:ECSCAPOLCNT + - \D3D::TOP.RF.ECH.SCARECROW:ECSCASTAT + - \D3D::TOP.RF.ECH.SCARECROW:ECSCATORCNT + - \D3D::TOP.RF.ECH.SCARECROW:ECSCAXMFRAC + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:SHOT_CALIB + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:CALC_FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:P1 + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:P2 + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:PCS + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:SHOT_CALIB + - \D3D::TOP.RF.ICH:ICHPWR + - \D3D::TOP.RF.ICH:ICHPWR:MULTIPLIER + - \D3D::TOP.RF.ICH:ICHPWR:UNITS + EFIT01: + - \EFIT01::AMINOR + - \EFIT01::ALPHA + - \EFIT01::BETAN + - \EFIT01::GAPIN + - \EFIT01::PSIRZ + - \EFIT01::PSIN + - \EFIT01::RHOVN + - \EFIT01::PRES + - \EFIT01::Q95 + - \EFIT01::QMIN + - \EFIT01::R0 + - \EFIT01::KAPPA + - \EFIT01::TRITOP + - \EFIT01::TRIBOT + MHD: + - \MHD::N1RMS + - \MHD::N2RMS + AOT: + - \AOT::TRIANGULARITY_U + - \AOT::TRIANGULARITY_L + - \AOT::Q + ptdata: + - MPI1A322D + - MPI3A322D + - MPI5A322D + - MPI89A322D + - MPI79FA322D + - MPI7FA322D + - MPI67A322D + - MPI6NA322D + - MPI1B322D + - MPI3B322D + - MPI5B322D + - MPI89B322D + - MPI79B322D + - MPI7NB322D + - MPI6FB322D + - MPI66M322D + - MPI66M132D + - MPI66B137D + - MPI66M312D + - MPI66B312D + - MPI66M020D + - MPI66M097D + - MPI66M307D + - MPI1A011D + - MPI1A274D + - MPI1A109D + - MPI1A199D + - MPI1A274D + - MPI1A341D + - b1 + - b2 + - b3 + - b4 + - b5 + - b6 + - b7 + - b8 + - TPLANG01 + - TPLANG02 + - TPLANG03 + - TPLANG04 + - TPLANG05 + - TPLANG06 + - TPLANG07 + - TPLANG08 + - TPLANG09 + - TPLANG10 + - TPLANG11 + - TPLANG12 + - TPLANG13 + - TPLANG14 + - TPLANG15 + - TPLANG16 + - TPLANG17 + - TPLANG18 + - TPLANG19 + - TPLANG20 + - TPLANG21 + - TPLANG22 + - TPLANG23 + - TPLANG24 + - TPLANG25 + - TPLANG26 + - TPLANG27 + - TPLANG28 + - TPLANG29 + - TPLANG30 + - TPLANG31 + - TPLANG32 + - TPLANG33 + - TPLANG34 + - TPLANG35 + - TPLANG36 + - TPLANG37 + - TPLANG38 + - TPLANG39 + - TPLANG40 + - TPLANG41 + - TPLANG42 + - TPLANG43 + - TPLANG44 + - TPLANG45 + - TPLANG46 + - TPLANG47 + - TPLANG48 + - TPLANG49 + - TPLANG50 + - TPLANG51 + - TPLANG52 + - TPLANG53 + - TPLANG54 + - TPLANG55 + - TPLANG56 + - TPLANG57 + - TPLANG58 + - TPLANG59 + - TPLANG60 + - TPLANG61 + - TPLANG62 + - TPLANG63 + - TPLANG64 + - TPLANG65 + - TPLANG66 + - TPLANG67 + - TPLANG68 + - TPLANG69 + - TPLANG70 + - TPLANG71 + - TPLANG72 + - C19F + - C79F + - C139F + - C199F + - C259F + - C319F + - IU30F + - IU90F + - IU150F + - IU210F + - IU270F + - IU330F + - IL30F + - IL90F + - IL150F + - IL210F + - IL270F + - IL330 + - BESFU01 + - BESFU02 + - BESFU03 + - BESFU04 + - BESFU05 + - BESFU06 + - BESFU07 + - BESFU08 + - BESFU09 + - BESFU10 + - BESFU11 + - BESFU12 + - BESFU13 + - BESFU14 + - BESFU15 + - BESFU16 + - BESFU17 + - BESFU18 + - BESFU19 + - BESFU20 + - BESFU21 + - BESFU22 + - BESFU23 + - BESFU24 + - BESFU25 + - BESFU26 + - BESFU27 + - BESFU28 + - BESFU29 + - BESFU30 + - BESFU31 + - BESFU32 + - BESFU33 + - BESFU34 + - BESFU35 + - BESFU36 + - BESFU37 + - BESFU38 + - BESFU39 + - BESFU40 + - BESFU41 + - BESFU42 + - BESFU43 + - BESFU44 + - BESFU45 + - BESFU46 + - BESFU47 + - BESFU48 + - BESFU49 + - BESFU50 + - BESFU51 + - BESFU52 + - BESFU53 + - BESFU54 + - BESFU55 + - BESFU56 + - BESFU57 + - BESFU58 + - BESFU59 + - BESFU60 + - BESFU61 + - BESFU62 + - BESFU63 + - BESFU64 + +server: atlas.gat.com diff --git a/scripts/data_fetching_omega/config_chiron.yaml b/scripts/data_fetching_omega/config_chiron.yaml new file mode 100644 index 0000000..ec4a554 --- /dev/null +++ b/scripts/data_fetching_omega/config_chiron.yaml @@ -0,0 +1,19 @@ +trees: + tangtv: + - \TANGTV::TOP.TANGTV:LODIV_240RM1:PAR:INTENSIFIED:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:LODIV_240RM1:PAR:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:LODIV_240RM1:PERP:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_225RP1:PERP:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_0RP1:PERP:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_225RP1:PAR:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_0RP1:PAR:STANDARD:VIDEO_IMAGES + irtv: + - \IRTV::TOP.IRTV:BIAS_105RM1:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:LOCEN_315RM1:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:LODIV_165RP2:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:LODIV_60RP2:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:PERI75R0:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:UPCEN_300RP1:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:UPDIV_225RM2:DIGITAL_CAM:DIGITAL_RAW + +server: chiron.gat.com diff --git a/scripts/data_fetching_omega/read_mds.sh b/scripts/data_fetching_omega/read_mds.sh new file mode 100644 index 0000000..5e564a9 --- /dev/null +++ b/scripts/data_fetching_omega/read_mds.sh @@ -0,0 +1,251 @@ +#!/bin/bash +#SBATCH --time=04:00:00 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=64G + +module load mdsplus + +# Configuration +CHUNK_SIZE=100 + +# Globus configuration +GLOBUS_SOURCE_ENDPOINT="20749357-d221-43c6-bbc4-79691e6776b8" +GLOBUS_DEST_ENDPOINT="544b12dc-cb3d-11e9-939b-02ff96a5aa76" +GLOBUS_DEST_PATH="/scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_time_series_data/" + +# Get shot number +SHOT_NUMBER=$(sed -n "${SLURM_ARRAY_TASK_ID}p" ${BATCH_FILE}) + +if [ -z "${SHOT_NUMBER}" ]; then + echo "ERROR: Could not get shot number for task ${SLURM_ARRAY_TASK_ID}" + exit 1 +fi + +echo "=========================================" +echo "Job started at: $(date)" +echo "Shot number: ${SHOT_NUMBER}" +echo "Config file: ${CONFIG_FILE}" +echo "Chunk size: ${CHUNK_SIZE}" +echo "=========================================" + +OUTPUT_FILE="${OUTPUT_DIR}/${SHOT_NUMBER}.h5" + +# Extract server +SERVER=$(grep "^server:" ${CONFIG_FILE} | cut -d: -f2- | xargs) + +# Create flat list: each line is "tree_name|signal_line" +TMP_FLAT_LIST=$(mktemp) + +awk ' +/^ [a-z0-9_]+:$/ { + current_tree = $1 + sub(/:$/, "", current_tree) + next +} +/^ - / { + if (current_tree != "") { + print current_tree "|" $0 + } +} +' ${CONFIG_FILE} > ${TMP_FLAT_LIST} + +TOTAL_SIGNALS=$(wc -l < ${TMP_FLAT_LIST}) +NUM_CHUNKS=$(( (TOTAL_SIGNALS + CHUNK_SIZE - 1) / CHUNK_SIZE )) + +echo "Total signals: ${TOTAL_SIGNALS}" +echo "Processing in ${NUM_CHUNKS} chunks" +echo "=========================================" + +FAILED_CHUNKS=0 + +for (( chunk=0; chunk "${CONFIG_FILE_CHUNK}" << EOF +shot_numbers: + - ${SHOT_NUMBER} + +trees: +EOF + + # Group signals by tree and add to config + echo "${CHUNK_DATA}" | awk -F'|' ' + { + tree = $1 + signal = $2 + if (tree != current_tree) { + if (current_tree != "") { + # Print accumulated signals for previous tree + for (i = 0; i < sig_count; i++) { + print signals[i] + } + } + # Start new tree + current_tree = tree + print " " tree ":" + sig_count = 0 + } + signals[sig_count++] = signal + } + END { + # Print last tree signals + if (sig_count > 0) { + for (i = 0; i < sig_count; i++) { + print signals[i] + } + } + } + ' >> "${CONFIG_FILE_CHUNK}" + + # Add output file and server + cat >> "${CONFIG_FILE_CHUNK}" << EOF + +out_filename: ${OUTPUT_FILE} +server: ${SERVER} +EOF + + # Run read_mds + echo " Running read_mds..." + read_mds -c ${CONFIG_FILE_CHUNK} + EXIT_CODE=$? + + if [ ${EXIT_CODE} -eq 0 ]; then + echo " ✓ Chunk ${CHUNK_NUM}/${NUM_CHUNKS} completed successfully" + rm -f ${CONFIG_FILE_CHUNK} + else + echo " ✗ Chunk ${CHUNK_NUM}/${NUM_CHUNKS} FAILED (exit code: ${EXIT_CODE})" + echo " Config preserved: ${CONFIG_FILE_CHUNK}" + FAILED_CHUNKS=$((FAILED_CHUNKS + 1)) + fi +done + +rm -f ${TMP_FLAT_LIST} + +echo "" +echo "=========================================" +echo "Processing summary:" +echo " Total signals: ${TOTAL_SIGNALS}" +echo " Total chunks: ${NUM_CHUNKS}" +echo " Failed chunks: ${FAILED_CHUNKS}" +echo "=========================================" + +# Check overall success +if [ ${FAILED_CHUNKS} -eq 0 ]; then + if [ -f "${OUTPUT_FILE}" ] && [ -s "${OUTPUT_FILE}" ]; then + echo "SUCCESS: All chunks completed, output file: ${OUTPUT_FILE}" + + ( + flock -x 200 + if ! grep -q "^${SHOT_NUMBER}$" ${COMPLETED_FILE} 2>/dev/null; then + echo "${SHOT_NUMBER}" >> ${COMPLETED_FILE} + fi + ) 200>${COMPLETED_FILE}.lock + + # ============================================ + # GLOBUS TRANSFER SECTION + # ============================================ + echo "" + echo "=========================================" + echo "Starting Globus transfer..." + + # Get relative path of the output file + OUTPUT_FILENAME=$(basename "${OUTPUT_FILE}") + + # Strip /cscratch/ from the path for Globus + # If OUTPUT_FILE="/cscratch/steinerp/database/data/170659.h5" + # Then GLOBUS_SOURCE_PATH="steinerp/database/data/170659.h5" + GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" + + # Transfer this file + echo "Transferring: ${OUTPUT_FILENAME}" + echo "Source path: ${GLOBUS_SOURCE_PATH}" + echo "Dest path: ${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}" + + TRANSFER_TASK_ID=$(globus transfer \ + --preserve-mtime \ + --label "Auto-transfer ${OUTPUT_FILENAME} $(date +%Y%m%d-%H%M%S)" \ + --jmespath 'task_id' \ + --format unix \ + --notify off \ + "${GLOBUS_SOURCE_ENDPOINT}:${GLOBUS_SOURCE_PATH}" \ + "${GLOBUS_DEST_ENDPOINT}:${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}") + + TRANSFER_EXIT_CODE=$? + echo "Transfer exit code: ${TRANSFER_EXIT_CODE}" + + if [ ${TRANSFER_EXIT_CODE} -eq 0 ]; then + echo "Transfer submitted: Task ID ${TRANSFER_TASK_ID}" + echo "Waiting for transfer to complete..." + + # Wait for transfer (with 2 hour timeout) + globus task wait "${TRANSFER_TASK_ID}" --timeout 7200 --polling-interval 30 + + if [ $? -eq 0 ]; then + echo "✓ Transfer completed successfully!" + echo "Deleting local file to free up space..." + + # Delete the transferred file + rm -f "${OUTPUT_FILE}" + + if [ $? -eq 0 ]; then + echo "✓ Local file deleted: ${OUTPUT_FILE}" + + # Log the transfer + TRANSFER_LOG="${OUTPUT_DIR}/globus_transfers.log" + echo "$(date '+%Y-%m-%d %H:%M:%S') | ${SHOT_NUMBER} | ${OUTPUT_FILENAME} | TRANSFERRED_AND_DELETED" >> ${TRANSFER_LOG} + else + echo "✗ WARNING: Could not delete local file" + fi + else + echo "✗ Transfer failed or timed out" + echo "Local file preserved: ${OUTPUT_FILE}" + fi + else + echo "✗ Transfer submission failed with exit code ${TRANSFER_EXIT_CODE}" + echo "Check: endpoint IDs, paths, and activation status" + fi + echo "=========================================" + # ============================================ + # END GLOBUS TRANSFER SECTION + # ============================================ + + echo "Job completed successfully at: $(date)" + exit 0 + else + echo "ERROR: Output file missing or empty: ${OUTPUT_FILE}" + FAILED_CHUNKS=1 + fi +fi + +echo "ERROR: ${FAILED_CHUNKS} chunk(s) failed for shot ${SHOT_NUMBER}" + +( + flock -x 200 + if ! grep -q "^${SHOT_NUMBER}$" ${FAILED_FILE} 2>/dev/null; then + echo "${SHOT_NUMBER}" >> ${FAILED_FILE} + fi +) 200>${FAILED_FILE}.lock + +echo "Job failed at: $(date)" +exit 1 diff --git a/scripts/data_fetching_omega/submit_read_mds_batches.sh b/scripts/data_fetching_omega/submit_read_mds_batches.sh new file mode 100644 index 0000000..bec9efa --- /dev/null +++ b/scripts/data_fetching_omega/submit_read_mds_batches.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +# ============================================ +# Configuration +# ============================================ +# Choose mode: "range" or "list" +MODE="list" # or "list" + +# For range mode: +SHOT_START=200700 +SHOT_END=200800 + +# For list mode (one shot number per line): +SHOT_LIST_FILE="shots_to_process.txt" + +# Common configuration +CONFIG_FILE="config_atlas.yaml" +OUTPUT_DIR="/cscratch/steinerp/database/data" +NODE_PATHS_DIR="/cscratch/steinerp/database/node_paths" # Deprecated but kept for compatibility + +# Batch settings +BATCH_SIZE=1000 +MAX_SUBMIT_LIMIT=25 + +# State files +STATE_FILE=".submission_state" +COMPLETED_FILE=".completed_shots" +FAILED_FILE=".failed_shots" + +# ============================================ +# Main Script +# ============================================ + +# Create output directory if it doesn't exist +mkdir -p ${OUTPUT_DIR} +mkdir -p jobs + +# Initialize tracking files if they don't exist +touch ${COMPLETED_FILE} +touch ${FAILED_FILE} + +echo "=========================================" +echo "MDSPlus Batch Data Fetcher" +echo "=========================================" +echo "Mode: ${MODE}" +echo "Config file: ${CONFIG_FILE}" + +if [ "${MODE}" = "range" ]; then + echo "Shot range: ${SHOT_START} to ${SHOT_END}" +elif [ "${MODE}" = "list" ]; then + echo "Shot list file: ${SHOT_LIST_FILE}" +else + echo "ERROR: Invalid MODE '${MODE}'. Must be 'range' or 'list'" + exit 1 +fi + +echo "Output directory: ${OUTPUT_DIR}" +echo "Batch size: ${BATCH_SIZE}" +echo "Max concurrent jobs: ${MAX_SUBMIT_LIMIT}" +echo "=========================================" + +# Generate shot list based on mode +SHOT_LIST=$(mktemp) + +if [ "${MODE}" = "range" ]; then + # Range mode: generate sequence + for shot in $(seq ${SHOT_START} ${SHOT_END}); do + # Skip if already completed + if grep -q "^${shot}$" ${COMPLETED_FILE} 2>/dev/null; then + continue + fi + echo ${shot} >> ${SHOT_LIST} + done + +elif [ "${MODE}" = "list" ]; then + # List mode: read from file + if [ ! -f "${SHOT_LIST_FILE}" ]; then + echo "ERROR: Shot list file not found: ${SHOT_LIST_FILE}" + rm -f ${SHOT_LIST} + exit 1 + fi + + # Read shots from file, skip already completed + while IFS= read -r shot; do + # Skip empty lines and comments + [[ -z "$shot" || "$shot" =~ ^[[:space:]]*# ]] && continue + + # Skip if already completed + if grep -q "^${shot}$" ${COMPLETED_FILE} 2>/dev/null; then + continue + fi + + echo ${shot} >> ${SHOT_LIST} + done < "${SHOT_LIST_FILE}" +fi + +TOTAL_SHOTS=$(wc -l < ${SHOT_LIST}) + +if [ ${TOTAL_SHOTS} -eq 0 ]; then + echo "No shots to process (all completed or none in range)" + rm -f ${SHOT_LIST} + exit 0 +fi + +echo "Total shots to process: ${TOTAL_SHOTS}" + +# Split into batches +BATCH_NUM=0 +SHOT_INDEX=0 + +while [ ${SHOT_INDEX} -lt ${TOTAL_SHOTS} ]; do + BATCH_NUM=$((BATCH_NUM + 1)) + BATCH_FILE="batch_${BATCH_NUM}.txt" + + # Extract batch + START_LINE=$((SHOT_INDEX + 1)) + END_LINE=$((SHOT_INDEX + BATCH_SIZE)) + + sed -n "${START_LINE},${END_LINE}p" ${SHOT_LIST} > ${BATCH_FILE} + + BATCH_SHOTS=$(wc -l < ${BATCH_FILE}) + + # Wait if queue is full + while true; do + RUNNING_JOBS=$(squeue -u $USER -h -t running,pending -r | wc -l) + + if [ ${RUNNING_JOBS} -lt ${MAX_SUBMIT_LIMIT} ]; then + break + fi + + # Get completion stats + COMPLETED_COUNT=$(wc -l < ${COMPLETED_FILE}) + FAILED_COUNT=$(wc -l < ${FAILED_FILE}) + + echo "Queue full: ${RUNNING_JOBS}/${MAX_SUBMIT_LIMIT} | Completed: ${COMPLETED_COUNT}/${TOTAL_SHOTS} | Failed: ${FAILED_COUNT}" + sleep 30 + done + + # Submit batch as job array + echo "Submitting batch ${BATCH_NUM} with ${BATCH_SHOTS} shots..." + + JOB_ID=$(sbatch --parsable \ + --array=1-${BATCH_SHOTS} \ + --output=jobs/job_%A_%a.out \ + --error=jobs/job_%A_%a.err \ + --export=ALL,BATCH_FILE=${BATCH_FILE},CONFIG_FILE=${CONFIG_FILE},OUTPUT_DIR=${OUTPUT_DIR},NODE_PATHS_DIR=${NODE_PATHS_DIR},COMPLETED_FILE=${COMPLETED_FILE},FAILED_FILE=${FAILED_FILE} \ + read_mds.sh) + + echo "Submitted batch ${BATCH_NUM} as job ${JOB_ID}" + + # Save state + echo "${BATCH_NUM}:${BATCH_FILE}:${JOB_ID}" >> ${STATE_FILE} + + SHOT_INDEX=$((SHOT_INDEX + BATCH_SHOTS)) + + # Brief pause between submissions + sleep 2 +done + +echo "" +echo "=========================================" +echo "All batches submitted (${BATCH_NUM} batches total)" +echo "Monitoring progress..." +echo "=========================================" + +# Monitor until completion +while true; do + RUNNING_JOBS=$(squeue -u $USER -h -t running,pending -r | wc -l) + COMPLETED_COUNT=$(wc -l < ${COMPLETED_FILE}) + FAILED_COUNT=$(wc -l < ${FAILED_FILE}) + + echo "Jobs running: ${RUNNING_JOBS} | Completed: ${COMPLETED_COUNT}/${TOTAL_SHOTS} | Failed: ${FAILED_COUNT}" + + if [ ${RUNNING_JOBS} -eq 0 ]; then + echo "All jobs finished" + break + fi + + sleep 60 +done + +echo "" +echo "=========================================" +echo "Final Summary" +echo "=========================================" +echo "Total shots requested: ${TOTAL_SHOTS}" +echo "Successfully completed: $(wc -l < ${COMPLETED_FILE})" +echo "Failed: $(wc -l < ${FAILED_FILE})" +echo "=========================================" + +# Cleanup +rm -f ${SHOT_LIST} +rm -f ${STATE_FILE} +rm -f batch_*.txt + +echo "Done at: $(date)" From ad3b3ff3276b909f0871a7915d270833cae0fc1a Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 24 Feb 2026 15:15:03 -0500 Subject: [PATCH 19/30] Added a documentation for setting up Globus CLI on Omega and start a simple file transfer. --- scripts/data_fetching_omega/README.md | 126 ++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 scripts/data_fetching_omega/README.md diff --git a/scripts/data_fetching_omega/README.md b/scripts/data_fetching_omega/README.md new file mode 100644 index 0000000..1a15594 --- /dev/null +++ b/scripts/data_fetching_omega/README.md @@ -0,0 +1,126 @@ +# Globus File Transfer Setup + +Automatic file transfer using Globus between Omega and Stellar clusters. + +## One-Time Setup + +### 1. Install Globus CLI + +```bash +module load mdsplus +pip3 install --user globus-cli +``` + +### 2. Authenticate + +```bash +globus login +``` + +Follow the URL, authenticate with your institution, and paste the authorization code back. + +### 3. Grant Collection Access + +Run for **both** source and destination collections: + +```bash +globus session consent 'urn:globus:auth:scope:transfer.api.globus.org:all[*https://auth.globus.org/scopes/COLLECTION_ID/data_access]' +``` + +Replace `COLLECTION_ID` with: +- Omega collection ID: `20749357-d221-43c6-bbc4-79691e6776b8` +- Stellar collection ID: `544b12dc-cb3d-11e9-939b-02ff96a5aa76` + +Or simply run `globus session update` and grant access when prompted. + +## Configuration + +### Find Collection IDs + +1. Go to https://app.globus.org/file-manager +2. Search for your collection +3. Copy the ID from the URL: `?origin_id=COLLECTION_ID` + +### Minimal Working Example + +```bash +#!/bin/bash + +module load mdsplus + +# Globus configuration +GLOBUS_SOURCE_ENDPOINT="20749357-d221-43c6-bbc4-79691e6776b8" # Omega +GLOBUS_DEST_ENDPOINT="544b12dc-cb3d-11e9-939b-02ff96a5aa76" # Stellar +GLOBUS_DEST_PATH="/scratch/gpfs/EKOLEMEN/big_d3d_data/" + +# Example file to transfer +OUTPUT_FILE="/cscratch/steinerp/database/data/example.h5" +OUTPUT_FILENAME=$(basename "${OUTPUT_FILE}") + +# Strip /cscratch/ mount point (Omega-specific) +GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" + +# Transfer +TRANSFER_TASK_ID=$(globus transfer \ + --preserve-mtime \ + --label "Transfer ${OUTPUT_FILENAME}" \ + --jmespath 'task_id' \ + --format unix \ + "${GLOBUS_SOURCE_ENDPOINT}:${GLOBUS_SOURCE_PATH}" \ + "${GLOBUS_DEST_ENDPOINT}:${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}") + +echo "Transfer submitted: ${TRANSFER_TASK_ID}" + +# Wait for completion +globus task wait "${TRANSFER_TASK_ID}" --timeout 7200 --polling-interval 30 + +# Delete local file after successful transfer (optional) +if [ $? -eq 0 ]; then + rm -f "${OUTPUT_FILE}" + echo "Transfer complete, local file deleted" +fi +``` + +## Important: Omega Mount Point + +The Omega Globus collection is mounted at `/cscratch/`. Always strip this prefix: + +```bash +# If OUTPUT_FILE="/cscratch/steinerp/data/file.h5" +GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" # becomes "steinerp/data/file.h5" +``` + +## Testing + +```bash +# Test access to both collections +globus ls 20749357-d221-43c6-bbc4-79691e6776b8:/steinerp/ +globus ls 544b12dc-cb3d-11e9-939b-02ff96a5aa76:/scratch/gpfs/EKOLEMEN/ + +# Test manual transfer +globus transfer \ + 20749357-d221-43c6-bbc4-79691e6776b8:steinerp/test.txt \ + 544b12dc-cb3d-11e9-939b-02ff96a5aa76:/scratch/gpfs/EKOLEMEN/test.txt +``` + +## Troubleshooting + +**"Missing required data_access consent"** + +```bash +globus session update +``` + +**Check transfer status** + +```bash +globus task list +globus task show TASK_ID +``` + +Or visit: https://app.globus.org/activity + +## Resources + +- [Globus Documentation](https://docs.globus.org/) +- [Globus CLI Reference](https://docs.globus.org/cli/) From ebe4c3adb93f4fdbbeef1de04b37ac53b175a382 Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 24 Feb 2026 16:03:02 -0500 Subject: [PATCH 20/30] Updated README.md: - Added information on how to use all the scripts for data fetching. Updated read_mds.sh - Added a switch for globus file transfer. This simply stores the H5 files on Omega and we can add more data later. --- scripts/data_fetching_omega/README.md | 360 +++++++++++++++++++----- scripts/data_fetching_omega/read_mds.sh | 113 ++++---- 2 files changed, 351 insertions(+), 122 deletions(-) diff --git a/scripts/data_fetching_omega/README.md b/scripts/data_fetching_omega/README.md index 1a15594..9bc2795 100644 --- a/scripts/data_fetching_omega/README.md +++ b/scripts/data_fetching_omega/README.md @@ -1,126 +1,346 @@ -# Globus File Transfer Setup +# MDSPlus Batch Data Fetcher -Automatic file transfer using Globus between Omega and Stellar clusters. +Automated framework for fetching large-scale MDSPlus data from DIII-D tokamak servers with optional Globus transfer to remote clusters. -## One-Time Setup +## Overview -### 1. Install Globus CLI +This framework: + +- Fetches MDSPlus data from multiple servers (atlas.gat.com, chiron.gat.com) +- Processes shots in parallel using SLURM job arrays +- Handles thousands of signals per shot via automatic chunking +- Optionally transfers files via Globus and cleans up local storage +- Tracks completion state for resume capability + +## File Structure + +``` +. +├── submit_read_mds_batches.sh # Main submission script +├── read_mds.sh # SLURM worker script +├── config_atlas.yaml # Signal list for atlas server +├── config_chiron.yaml # Signal list for chiron server +├── README.md # This file +├── .completed_shots # Auto-generated: completed shots +├── .failed_shots # Auto-generated: failed shots +└── jobs/ # Auto-generated: job logs +``` + +## Quick Start + +### 1. Configure Shot Range or List + +Edit `submit_read_mds_batches.sh`: ```bash -module load mdsplus -pip3 install --user globus-cli +# Option A: Process a range of shots +MODE="range" +SHOT_START=200000 +SHOT_END=200100 + +# Option B: Process shots from a file +MODE="list" +SHOT_LIST_FILE="shots_to_process.txt" ``` -### 2. Authenticate +### 2. Select Configuration ```bash -globus login +# Choose which server/signals to fetch +CONFIG_FILE="config_atlas.yaml" # or config_chiron.yaml ``` -Follow the URL, authenticate with your institution, and paste the authorization code back. +### 3. Configure Output -### 3. Grant Collection Access +```bash +# Where to save HDF5 files +OUTPUT_DIR="/cscratch/steinerp/database/data" -Run for **both** source and destination collections: +# Batch settings +BATCH_SIZE=1000 # Shots per batch +MAX_SUBMIT_LIMIT=25 # Max concurrent jobs +``` + +### 4. Configure Globus (Optional) + +Edit `read_mds.sh`: ```bash -globus session consent 'urn:globus:auth:scope:transfer.api.globus.org:all[*https://auth.globus.org/scopes/COLLECTION_ID/data_access]' +# Enable/disable automatic transfer +ENABLE_GLOBUS=true # Set to false to keep files locally + +# Globus endpoints (if enabled) +GLOBUS_SOURCE_ENDPOINT="your-source-id" +GLOBUS_DEST_ENDPOINT="your-dest-id" +GLOBUS_DEST_PATH="/path/on/destination/" ``` -Replace `COLLECTION_ID` with: -- Omega collection ID: `20749357-d221-43c6-bbc4-79691e6776b8` -- Stellar collection ID: `544b12dc-cb3d-11e9-939b-02ff96a5aa76` +### 5. Submit Jobs -Or simply run `globus session update` and grant access when prompted. +**Option A: Run in foreground (blocks terminal)** -## Configuration +```bash +./submit_read_mds_batches.sh +``` -### Find Collection IDs +**Option B: Run in background with nohup (recommended for long runs)** -1. Go to https://app.globus.org/file-manager -2. Search for your collection -3. Copy the ID from the URL: `?origin_id=COLLECTION_ID` +```bash +nohup ./submit_read_mds_batches.sh > submission_d3d_mdsplus.log 2>&1 & +``` -### Minimal Working Example +This will: +- Run in background (terminal can be closed) +- Write all output to `submission_d3d_mdsplus.log` +- Return immediately with process ID + +**Monitor background job:** ```bash -#!/bin/bash +# Check if still running +ps aux | grep submit_read_mds_batches.sh -module load mdsplus +# View progress +tail -f submission_d3d_mdsplus.log -# Globus configuration -GLOBUS_SOURCE_ENDPOINT="20749357-d221-43c6-bbc4-79691e6776b8" # Omega -GLOBUS_DEST_ENDPOINT="544b12dc-cb3d-11e9-939b-02ff96a5aa76" # Stellar -GLOBUS_DEST_PATH="/scratch/gpfs/EKOLEMEN/big_d3d_data/" +# Check completion +grep "Final Summary" submission_d3d_mdsplus.log +``` + +## Configuration Files + +### Signal Configuration (YAML) + +```yaml +trees: + d3d: + - \D3D::TOP.MAGNETICS.BPOL_PROBE:BP01 + - \D3D::TOP.MAGNETICS.BPOL_PROBE:BP02 + ptdata: + - \PTDATA::TOP.RESULTS.ETEMP_PROFILE + +server: atlas.gat.com +``` -# Example file to transfer -OUTPUT_FILE="/cscratch/steinerp/database/data/example.h5" -OUTPUT_FILENAME=$(basename "${OUTPUT_FILE}") +- **trees**: Groups signals by MDSPlus tree +- **signals**: Full MDSPlus paths (one per line) +- **server**: MDSPlus server hostname -# Strip /cscratch/ mount point (Omega-specific) -GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" +### Shot List File -# Transfer -TRANSFER_TASK_ID=$(globus transfer \ - --preserve-mtime \ - --label "Transfer ${OUTPUT_FILENAME}" \ - --jmespath 'task_id' \ - --format unix \ - "${GLOBUS_SOURCE_ENDPOINT}:${GLOBUS_SOURCE_PATH}" \ - "${GLOBUS_DEST_ENDPOINT}:${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}") +Create `shots_to_process.txt`: -echo "Transfer submitted: ${TRANSFER_TASK_ID}" +``` +# Campaign 2025 shots +200000 +200015 +200032 + +# Failed shots to retry +200100 +200250 +``` + +- One shot number per line +- Lines starting with `#` are comments +- Empty lines ignored -# Wait for completion -globus task wait "${TRANSFER_TASK_ID}" --timeout 7200 --polling-interval 30 +## Output Structure -# Delete local file after successful transfer (optional) -if [ $? -eq 0 ]; then - rm -f "${OUTPUT_FILE}" - echo "Transfer complete, local file deleted" -fi ``` +HDF5_FILE.h5 +├── 200000/ # Shot number +│ ├── d3d/ # Tree name +│ │ ├── \D3D::TOP.SIGNAL/ +│ │ │ ├── data # Signal values +│ │ │ └── dim0 # Time axis +``` + +## Features + +### Automatic Chunking + +Large signal lists are automatically split into chunks (default: 100 signals/chunk) to avoid "Argument list too long" errors. + +### State Tracking + +- `.completed_shots` - Successfully processed shots (skipped on restart) +- `.failed_shots` - Failed shots for review +- Locked file writes prevent race conditions + +### Resume Capability + +Rerun `submit_read_mds_batches.sh` to: + +- Skip already completed shots +- Retry only failed shots +- Continue interrupted processing + +### Globus Transfer + +When `ENABLE_GLOBUS=true`: + +1. File is transferred to remote cluster +2. Transfer completion is verified +3. Local file is deleted to save space +4. Transfer logged to `globus_transfers.log` + +When `ENABLE_GLOBUS=false`: + +- Files remain in `OUTPUT_DIR` +- No automatic cleanup -## Important: Omega Mount Point +## Monitoring -The Omega Globus collection is mounted at `/cscratch/`. Always strip this prefix: +### Check Progress ```bash -# If OUTPUT_FILE="/cscratch/steinerp/data/file.h5" -GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" # becomes "steinerp/data/file.h5" +# View current status +tail -f jobs/job_*.out + +# Count completed/failed +wc -l .completed_shots .failed_shots + +# Check queue +squeue -u $USER ``` -## Testing +### View Logs ```bash -# Test access to both collections -globus ls 20749357-d221-43c6-bbc4-79691e6776b8:/steinerp/ -globus ls 544b12dc-cb3d-11e9-939b-02ff96a5aa76:/scratch/gpfs/EKOLEMEN/ +# Latest job output +ls -t jobs/job_*.out | head -1 | xargs cat -# Test manual transfer -globus transfer \ - 20749357-d221-43c6-bbc4-79691e6776b8:steinerp/test.txt \ - 544b12dc-cb3d-11e9-939b-02ff96a5aa76:/scratch/gpfs/EKOLEMEN/test.txt +# Failed shots +cat .failed_shots ``` ## Troubleshooting -**"Missing required data_access consent"** +### No Shots Processed + +**Problem**: `No shots to process (all completed or none in range)` + +**Solutions**: + +- Check shot range: `SHOT_START` and `SHOT_END` +- Verify shots aren't in `.completed_shots` +- For list mode: check `SHOT_LIST_FILE` exists and contains shots + +### Chunk Failures + +**Problem**: `Chunk X/Y FAILED` + +**Solutions**: + +- Check preserved config: `config_SHOT_chunkN_*.yml` +- Verify server connectivity: `ping atlas.gat.com` +- Check signal paths in config file +- Review job logs in `jobs/` directory + +### Globus Errors + +**Problem**: `Transfer submission failed` + +**Solutions**: + +- Verify endpoints are activated +- Check endpoint IDs are correct +- Ensure collection paths are accessible +- Re-authenticate: `globus login` +- Grant data access (see Globus setup below) + +### Memory Errors + +**Problem**: `Out of memory` + +**Solutions**: + +- Reduce `CHUNK_SIZE` in `read_mds.sh` (default: 100) +- Increase memory: `#SBATCH --mem=128G` +- Process fewer signals per config + +## Globus Setup + +### One-Time Setup + +```bash +# Install Globus CLI +module load mdsplus +pip3 install globus-cli + +# Authenticate +globus login + +# Grant collection access +globus session consent 'urn:globus:auth:scope:transfer.api.globus.org:all[*https://auth.globus.org/scopes/COLLECTION_ID/data_access]' +``` + +### Find Endpoint IDs + +1. Go to https://app.globus.org/file-manager +2. Select your collection +3. Copy ID from URL: `?origin_id=ENDPOINT_ID` + +### Test Transfer ```bash -globus session update +globus ls ENDPOINT_ID:/path/to/files/ +globus transfer SOURCE_ID:/path/file.h5 DEST_ID:/path/file.h5 ``` -**Check transfer status** +## Advanced Usage + +### Process Specific Shots ```bash -globus task list -globus task show TASK_ID +# Create shot list +echo -e "200000\n200015\n200032" > my_shots.txt + +# Configure +MODE="list" +SHOT_LIST_FILE="my_shots.txt" + +# Submit +./submit_read_mds_batches.sh ``` -Or visit: https://app.globus.org/activity +### Retry Failed Shots + +```bash +# Use failed shots as input +cp .failed_shots shots_to_retry.txt + +# Clear failed list +> .failed_shots + +# Configure and submit +MODE="list" +SHOT_LIST_FILE="shots_to_retry.txt" +./submit_read_mds_batches.sh +``` + +### Multiple Configurations + +```bash +# Submit atlas jobs +CONFIG_FILE="config_atlas.yaml" +./submit_read_mds_batches.sh & + +# Submit chiron jobs +CONFIG_FILE="config_chiron.yaml" +./submit_read_mds_batches.sh & +``` + +## Performance Tips + +- **Chunk size**: Smaller = more overhead, larger = higher memory +- **Batch size**: Balance between queue management and parallelism +- **Max jobs**: Respect cluster limits +- **Globus**: Disable if processing locally or transferring later -## Resources +## Support -- [Globus Documentation](https://docs.globus.org/) -- [Globus CLI Reference](https://docs.globus.org/cli/) +For issues: +1. Check job logs: `jobs/job_*.err` +2. Check Globus status: https://app.globus.org/activity diff --git a/scripts/data_fetching_omega/read_mds.sh b/scripts/data_fetching_omega/read_mds.sh index 5e564a9..0b0dda7 100644 --- a/scripts/data_fetching_omega/read_mds.sh +++ b/scripts/data_fetching_omega/read_mds.sh @@ -10,6 +10,7 @@ module load mdsplus CHUNK_SIZE=100 # Globus configuration +ENABLE_GLOBUS=true # Set to false to disable Globus transfer GLOBUS_SOURCE_ENDPOINT="20749357-d221-43c6-bbc4-79691e6776b8" GLOBUS_DEST_ENDPOINT="544b12dc-cb3d-11e9-939b-02ff96a5aa76" GLOBUS_DEST_PATH="/scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_time_series_data/" @@ -165,68 +166,76 @@ if [ ${FAILED_CHUNKS} -eq 0 ]; then # ============================================ # GLOBUS TRANSFER SECTION # ============================================ - echo "" - echo "=========================================" - echo "Starting Globus transfer..." + if [ "${ENABLE_GLOBUS}" = true ]; then + echo "" + echo "=========================================" + echo "Starting Globus transfer..." + + # Get relative path of the output file + OUTPUT_FILENAME=$(basename "${OUTPUT_FILE}") + + # Strip /cscratch/ from the path for Globus + # If OUTPUT_FILE="/cscratch/steinerp/database/data/170659.h5" + # Then GLOBUS_SOURCE_PATH="steinerp/database/data/170659.h5" + GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" + + # Transfer this file + echo "Transferring: ${OUTPUT_FILENAME}" + echo "Source path: ${GLOBUS_SOURCE_PATH}" + echo "Dest path: ${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}" + + TRANSFER_TASK_ID=$(globus transfer \ + --preserve-mtime \ + --label "Auto-transfer ${OUTPUT_FILENAME} $(date +%Y%m%d-%H%M%S)" \ + --jmespath 'task_id' \ + --format unix \ + --notify off \ + "${GLOBUS_SOURCE_ENDPOINT}:${GLOBUS_SOURCE_PATH}" \ + "${GLOBUS_DEST_ENDPOINT}:${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}") + + TRANSFER_EXIT_CODE=$? + echo "Transfer exit code: ${TRANSFER_EXIT_CODE}" + + if [ ${TRANSFER_EXIT_CODE} -eq 0 ]; then + echo "Transfer submitted: Task ID ${TRANSFER_TASK_ID}" + echo "Waiting for transfer to complete..." + + # Wait for transfer (with 2 hour timeout) + globus task wait "${TRANSFER_TASK_ID}" --timeout 7200 --polling-interval 30 - # Get relative path of the output file - OUTPUT_FILENAME=$(basename "${OUTPUT_FILE}") - - # Strip /cscratch/ from the path for Globus - # If OUTPUT_FILE="/cscratch/steinerp/database/data/170659.h5" - # Then GLOBUS_SOURCE_PATH="steinerp/database/data/170659.h5" - GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" - - # Transfer this file - echo "Transferring: ${OUTPUT_FILENAME}" - echo "Source path: ${GLOBUS_SOURCE_PATH}" - echo "Dest path: ${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}" - - TRANSFER_TASK_ID=$(globus transfer \ - --preserve-mtime \ - --label "Auto-transfer ${OUTPUT_FILENAME} $(date +%Y%m%d-%H%M%S)" \ - --jmespath 'task_id' \ - --format unix \ - --notify off \ - "${GLOBUS_SOURCE_ENDPOINT}:${GLOBUS_SOURCE_PATH}" \ - "${GLOBUS_DEST_ENDPOINT}:${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}") - - TRANSFER_EXIT_CODE=$? - echo "Transfer exit code: ${TRANSFER_EXIT_CODE}" - - if [ ${TRANSFER_EXIT_CODE} -eq 0 ]; then - echo "Transfer submitted: Task ID ${TRANSFER_TASK_ID}" - echo "Waiting for transfer to complete..." - - # Wait for transfer (with 2 hour timeout) - globus task wait "${TRANSFER_TASK_ID}" --timeout 7200 --polling-interval 30 + if [ $? -eq 0 ]; then + echo "✓ Transfer completed successfully!" + echo "Deleting local file to free up space..." - if [ $? -eq 0 ]; then - echo "✓ Transfer completed successfully!" - echo "Deleting local file to free up space..." + # Delete the transferred file + rm -f "${OUTPUT_FILE}" - # Delete the transferred file - rm -f "${OUTPUT_FILE}" + if [ $? -eq 0 ]; then + echo "✓ Local file deleted: ${OUTPUT_FILE}" - if [ $? -eq 0 ]; then - echo "✓ Local file deleted: ${OUTPUT_FILE}" - - # Log the transfer - TRANSFER_LOG="${OUTPUT_DIR}/globus_transfers.log" - echo "$(date '+%Y-%m-%d %H:%M:%S') | ${SHOT_NUMBER} | ${OUTPUT_FILENAME} | TRANSFERRED_AND_DELETED" >> ${TRANSFER_LOG} + # Log the transfer + TRANSFER_LOG="${OUTPUT_DIR}/globus_transfers.log" + echo "$(date '+%Y-%m-%d %H:%M:%S') | ${SHOT_NUMBER} | ${OUTPUT_FILENAME} | TRANSFERRED_AND_DELETED" >> ${TRANSFER_LOG} + else + echo "✗ WARNING: Could not delete local file" + fi else - echo "✗ WARNING: Could not delete local file" + echo "✗ Transfer failed or timed out" + echo "Local file preserved: ${OUTPUT_FILE}" fi else - echo "✗ Transfer failed or timed out" - echo "Local file preserved: ${OUTPUT_FILE}" + echo "✗ Transfer submission failed with exit code ${TRANSFER_EXIT_CODE}" + echo "Check: endpoint IDs, paths, and activation status" fi + echo "=========================================" else - echo "✗ Transfer submission failed with exit code ${TRANSFER_EXIT_CODE}" - echo "Check: endpoint IDs, paths, and activation status" + echo "" + echo "=========================================" + echo "Globus transfer disabled - file retained locally" + echo "File location: ${OUTPUT_FILE}" + echo "=========================================" fi - echo "=========================================" - # ============================================ + # ============================================ # END GLOBUS TRANSFER SECTION # ============================================ From aa78e87e255f9a8de2f96d929c2234e5fad7eff7 Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 24 Feb 2026 17:01:29 -0500 Subject: [PATCH 21/30] More PTData to fetch. --- scripts/data_fetching_omega/config_atlas.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/data_fetching_omega/config_atlas.yaml b/scripts/data_fetching_omega/config_atlas.yaml index 76a536d..4771c7b 100644 --- a/scripts/data_fetching_omega/config_atlas.yaml +++ b/scripts/data_fetching_omega/config_atlas.yaml @@ -1844,5 +1844,17 @@ trees: - BESFU62 - BESFU63 - BESFU64 + - bcoil + - bmspinj + - bmstinj + - bt + - dssdenest + - fzns + - ip + - ipsip + - iptipp + - pcbcoil + - plasticfix + - dstdenp server: atlas.gat.com From e6cb74fa9d284d04081097084105ff9e3145e8fa Mon Sep 17 00:00:00 2001 From: renierts Date: Wed, 25 Feb 2026 13:46:28 -0500 Subject: [PATCH 22/30] PEP-8 compatible code. Moved prepare_data.py to scripts, added a batch script to do this on compute nodes. Added more point names to the data fetching scripts for Omega. Added docstring to the WelfordTensor class. Updated modalities.yaml with the new point names added. --- pixi.lock | 710 +++------- pyproject.toml | 6 +- scripts/data_fetching_omega/config_atlas.yaml | 59 + scripts/data_fetching_omega/merge_h5.py | 183 +++ scripts/data_preparation/prepare_data.py | 721 ++++++++++ scripts/slurm/benchmark_data_loader.sh | 12 + scripts/slurm/make_processing_stats.sh | 12 + scripts/slurm/prepare_data.sh | 12 + scripts/training/benchmark_data_loader.py | 44 + scripts/training/profile_reconstruction.py | 1 - .../data/config/config.yaml | 2 +- .../data/config/modalities/modalities.yaml | 1254 ++++++++++++++++- .../data/data_loader.py | 169 ++- .../data/prepare_data.py | 247 ---- 14 files changed, 2536 insertions(+), 896 deletions(-) create mode 100644 scripts/data_fetching_omega/merge_h5.py create mode 100644 scripts/data_preparation/prepare_data.py create mode 100755 scripts/slurm/benchmark_data_loader.sh create mode 100755 scripts/slurm/make_processing_stats.sh create mode 100755 scripts/slurm/prepare_data.sh create mode 100644 scripts/training/benchmark_data_loader.py delete mode 100644 src/tokamak_foundation_model/data/prepare_data.py diff --git a/pixi.lock b/pixi.lock index 161a9be..c7e0438 100644 --- a/pixi.lock +++ b/pixi.lock @@ -15,30 +15,22 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.20-py311hc665b79_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_101.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-5_h4a7cf45_openblas.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-5_h0358290_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.7.3-hecca717_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-15.2.0-he0feb66_17.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_17.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_17.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-h68bc16d_17.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_17.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-5_h47877c9_openblas.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.30-pthreads_h94d23a6_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.51.2-hf4e2dac_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_17.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.4.2-py311h2e04523_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.1-h35e630c_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda @@ -46,7 +38,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pyyaml-6.0.3-py311h3778330_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.3-h853b02a_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/scipy-1.17.0-py311hbe70eeb_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda @@ -64,6 +55,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/0b/02/4dbe7568a42e46582248942f54dc64ad094769532adbe21e525e4edf7bc4/cuda_pathfinder-1.3.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl @@ -98,6 +90,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4c/1a/edbe839109518364ac0bd9e918cf874c755bb2c128040e920f198c494263/numexpr-2.14.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/1b/46/6fa4ea94f1ddf969b2ee941290cca6f1bfac92b53c76ae5f44afe17ceb69/numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl - pypi: https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl @@ -131,6 +124,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/ef/df/df1457c4df3826e908879fe3d76bc5b6e60aae45f4ee42539512438cfd5d/scipy-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl @@ -152,29 +146,17 @@ environments: - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ osx-arm64: - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/_openmp_mutex-4.5-7_kmp_llvm.conda - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/debugpy-1.8.20-py311h8948835_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.2-h38cb7af_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libblas-3.11.0-5_h51639a9_openblas.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcblas-3.11.0-5_hb0561ab_openblas.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-h55c6f16_2.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-hcf2aa1b_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgcc-15.2.0-hcbb3090_17.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-h07b0088_17.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-hdae7583_17.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.11.0-5_hd9741b5_openblas.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.2-h8088a28_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenblas-0.3.30-openmp_ha158390_4.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.2-h1ae2325_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-21.1.8-h4a912ad_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/numpy-2.4.2-py311had1e860_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.1-hd24854e_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda @@ -182,7 +164,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/pyyaml-6.0.3-py311hc290fe0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.3-h46df422_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/scipy-1.17.0-py311he9931d0_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h010d191_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda @@ -198,6 +179,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl @@ -232,6 +214,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/95/d64f680ea1fc56d165457287e0851d6708800f9fcea346fc1b9957942ee6/numexpr-2.14.1-cp311-cp311-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/dd/5e/e04a547ad0f0183bf151fd7c7a477468e3b85ff2ad231c566389e6cc9587/pandas-3.0.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl @@ -250,6 +233,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/5e/5f/a6b38f79a07d74989224d5f11b55267714707582908a5f1ae854cf9a9b84/scipy-1.17.0-cp311-cp311-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl @@ -273,33 +257,18 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-h4c7d964_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/debugpy-1.8.20-py311h5dfdfe8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/icu-78.2-h637d24d_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libblas-3.11.0-5_hf2e6a31_mkl.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libcblas-3.11.0-5_h2a3cdd5_mkl.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libexpat-2.7.3-hac47afa_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libffi-3.5.2-h3d046cb_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.2-default_h4379cf1_1000.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.11.0-5_hf9ab0e9_mkl.conda - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.2-hfd05255_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.2-hf5d6505_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libwinpthread-12.0.0.r4.gg4f2fc60ca-h57928b3_10.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-16-2.15.1-h3cfd58e_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-2.15.1-h779ef1b_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/llvm-openmp-21.1.8-h4fa8253_0.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/mkl-2025.3.0-hac47afa_455.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/numpy-2.4.2-py311h80b3fa1_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.1-hf411b9b_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/python-3.11.14-h0159041_3_cpython.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.11-8_cp311.conda - conda: https://conda.anaconda.org/conda-forge/win-64/pyyaml-6.0.3-py311h3f79411_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/scipy-1.17.0-py311h9c22a71_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/tbb-2022.3.0-h3155e25_2.conda - conda: https://conda.anaconda.org/conda-forge/win-64/tk-8.6.13-h6ed50ae_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda @@ -319,6 +288,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/d5/92/1cb532e88560cbee973396254b21bece8c5d7c2ece958a67afa08c9f10dc/debugpy-1.8.20-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl @@ -353,6 +323,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/72/4ca9bd97b2eb6dce9f5e70a3b6acec1a93e1fb9b079cb4cba2cdfbbf295d/numexpr-2.14.1-cp311-cp311-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/51/27/bf9436dd0a4fc3130acec0828951c7ef96a0631969613a9a35744baf27f6/pandas-3.0.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl @@ -369,6 +340,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/52/c8/08629657ac6c0da198487ce8cd3de78e02cfde42b7f34117d56a3fe249dc/scipy-1.17.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl @@ -732,17 +704,6 @@ packages: purls: [] size: 23621 timestamp: 1650670423406 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/_openmp_mutex-4.5-7_kmp_llvm.conda - build_number: 7 - sha256: 7acaa2e0782cad032bdaf756b536874346ac1375745fb250e9bdd6a48a7ab3cd - md5: a44032f282e7d2acdeb1c240308052dd - depends: - - llvm-openmp >=9.0.1 - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 8325 - timestamp: 1764092507920 - conda: https://conda.anaconda.org/conda-forge/noarch/aiohappyeyeballs-2.6.1-pyhd8ed1ab_0.conda sha256: 7842ddc678e77868ba7b92a726b437575b23aaec293bca0d40826f1026d90e27 md5: 18fd895e0e775622906cdabfc3cf0fb4 @@ -1689,6 +1650,16 @@ packages: - pytest-cov ; extra == 'tests' - pytest-xdist ; extra == 'tests' requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/d5/92/1cb532e88560cbee973396254b21bece8c5d7c2ece958a67afa08c9f10dc/debugpy-1.8.20-cp311-cp311-win_amd64.whl + name: debugpy + version: 1.8.20 + sha256: 1f7650546e0eded1902d0f6af28f787fa1f1dbdbc97ddabaf1cd963a405930cb + requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/e0/c3/7f67dea8ccf8fdcb9c99033bbe3e90b9e7395415843accb81428c441be2d/debugpy-1.8.20-py2.py3-none-any.whl + name: debugpy + version: 1.8.20 + sha256: 5be9bed9ae3be00665a06acaa48f8329d2b9632f15fd09f6a9a8c8d9907e54d7 + requires_python: '>=3.8' - conda: https://conda.anaconda.org/conda-forge/linux-64/debugpy-1.8.20-py311hc665b79_0.conda sha256: e69be2be543c4d4898895d8aebe758bc683c5a1198583ad676f5719782a07131 md5: 400e4667a12884216df869cad5fb004b @@ -1704,36 +1675,6 @@ packages: - pkg:pypi/debugpy?source=hash-mapping size: 2733654 timestamp: 1769744984842 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/debugpy-1.8.20-py311h8948835_0.conda - sha256: 093b015e9abf27fb4d3b4f7e52417d35cd69a99fab8b95ec5c6c3983275c46ba - md5: 150c921424bc9f08c0378f8a6ae58d05 - depends: - - python - - __osx >=11.0 - - libcxx >=19 - - python 3.11.* *_cpython - - python_abi 3.11.* *_cp311 - license: MIT - license_family: MIT - purls: - - pkg:pypi/debugpy?source=hash-mapping - size: 2668163 - timestamp: 1769745020016 -- conda: https://conda.anaconda.org/conda-forge/win-64/debugpy-1.8.20-py311h5dfdfe8_0.conda - sha256: 661e5c582b1f853a46a78d4bb6e55f2bfdac66e68d015e111f1580a11c28abbf - md5: 683be2cd10e80a367790b3083ce529b7 - depends: - - python - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - - ucrt >=10.0.20348.0 - - python_abi 3.11.* *_cp311 - license: MIT - license_family: MIT - purls: - - pkg:pypi/debugpy?source=hash-mapping - size: 3940002 - timestamp: 1769745017274 - pypi: https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl name: decorator version: 5.2.1 @@ -1828,7 +1769,7 @@ packages: - pypi: ./ name: faith version: 26.1.dev0 - sha256: d143d15dacb53dea0f310e30e110adc36cded0de714eedb798a1145ffea4c3ea + sha256: 947201fad263cc81e9052dd4afa8eef157340bf2839eae66cbb7558ce7d0d073 requires_dist: - einops>=0.8.2,<0.9 - h5py>=3.15.1,<4 @@ -1837,6 +1778,7 @@ packages: - matplotlib>=3.10.8,<4 - numpy>=1.26.4,<3 - pandas>=3.0.0,<4 + - scipy - tables>=3.10.2,<4 - torch - torchinfo>=1.8.0,<2 @@ -2482,18 +2424,6 @@ packages: purls: [] size: 12358010 timestamp: 1767970350308 -- conda: https://conda.anaconda.org/conda-forge/win-64/icu-78.2-h637d24d_0.conda - sha256: 5a41fb28971342e293769fc968b3414253a2f8d9e30ed7c31517a15b4887246a - md5: 0ee3bb487600d5e71ab7d28951b2016a - depends: - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: MIT - license_family: MIT - purls: [] - size: 13222158 - timestamp: 1767970128854 - pypi: https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl name: idna version: '3.11' @@ -3355,24 +3285,6 @@ packages: purls: [] size: 483116 timestamp: 1759482133380 -- conda: https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-5_h4a7cf45_openblas.conda - build_number: 5 - sha256: 18c72545080b86739352482ba14ba2c4815e19e26a7417ca21a95b76ec8da24c - md5: c160954f7418d7b6e87eaf05a8913fa9 - depends: - - libopenblas >=0.3.30,<0.3.31.0a0 - - libopenblas >=0.3.30,<1.0a0 - constrains: - - mkl <2026 - - liblapack 3.11.0 5*_openblas - - libcblas 3.11.0 5*_openblas - - blas 2.305 openblas - - liblapacke 3.11.0 5*_openblas - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 18213 - timestamp: 1765818813880 - conda: https://conda.anaconda.org/conda-forge/linux-64/libblas-3.11.0-7_hc00574d_netlib.conda build_number: 7 sha256: 464608528e7b188fa3a602c503c7f73b3b446bbfd7b259d1c8b56470c34166fc @@ -3392,40 +3304,6 @@ packages: purls: [] size: 222771 timestamp: 1763440535188 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libblas-3.11.0-5_h51639a9_openblas.conda - build_number: 5 - sha256: 620a6278f194dcabc7962277da6835b1e968e46ad0c8e757736255f5ddbfca8d - md5: bcc025e2bbaf8a92982d20863fe1fb69 - depends: - - libopenblas >=0.3.30,<0.3.31.0a0 - - libopenblas >=0.3.30,<1.0a0 - constrains: - - libcblas 3.11.0 5*_openblas - - liblapack 3.11.0 5*_openblas - - liblapacke 3.11.0 5*_openblas - - blas 2.305 openblas - - mkl <2026 - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 18546 - timestamp: 1765819094137 -- conda: https://conda.anaconda.org/conda-forge/win-64/libblas-3.11.0-5_hf2e6a31_mkl.conda - build_number: 5 - sha256: f0cb7b2697461a306341f7ff32d5b361bb84f3e94478464c1e27ee01fc8f276b - md5: f9decf88743af85c9c9e05556a4c47c0 - depends: - - mkl >=2025.3.0,<2026.0a0 - constrains: - - liblapack 3.11.0 5*_mkl - - libcblas 3.11.0 5*_mkl - - blas 2.305 mkl - - liblapacke 3.11.0 5*_mkl - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 67438 - timestamp: 1765819100043 - conda: https://conda.anaconda.org/conda-forge/linux-64/libbrotlicommon-1.1.0-hb03c661_4.conda sha256: 2338a92d1de71f10c8cf70f7bb9775b0144a306d75c4812276749f54925612b6 md5: 1d29d2e33fe59954af82ef54a8af3fe1 @@ -3461,21 +3339,6 @@ packages: purls: [] size: 289680 timestamp: 1756599375485 -- conda: https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-5_h0358290_openblas.conda - build_number: 5 - sha256: 0cbdcc67901e02dc17f1d19e1f9170610bd828100dc207de4d5b6b8ad1ae7ad8 - md5: 6636a2b6f1a87572df2970d3ebc87cc0 - depends: - - libblas 3.11.0 5_h4a7cf45_openblas - constrains: - - liblapacke 3.11.0 5*_openblas - - blas 2.305 openblas - - liblapack 3.11.0 5*_openblas - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 18194 - timestamp: 1765818837135 - conda: https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.11.0-7_h8e06fc2_netlib.conda build_number: 7 sha256: 7940cc63673587cb7946831431b0527ce5707e24a54df87644c199e40c2714b4 @@ -3494,36 +3357,6 @@ packages: purls: [] size: 50122 timestamp: 1763440541127 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcblas-3.11.0-5_hb0561ab_openblas.conda - build_number: 5 - sha256: 38809c361bbd165ecf83f7f05fae9b791e1baa11e4447367f38ae1327f402fc0 - md5: efd8bd15ca56e9d01748a3beab8404eb - depends: - - libblas 3.11.0 5_h51639a9_openblas - constrains: - - liblapacke 3.11.0 5*_openblas - - liblapack 3.11.0 5*_openblas - - blas 2.305 openblas - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 18548 - timestamp: 1765819108956 -- conda: https://conda.anaconda.org/conda-forge/win-64/libcblas-3.11.0-5_h2a3cdd5_mkl.conda - build_number: 5 - sha256: 49dc59d8e58360920314b8d276dd80da7866a1484a9abae4ee2760bc68f3e68d - md5: b3fa8e8b55310ba8ef0060103afb02b5 - depends: - - libblas 3.11.0 5_hf2e6a31_mkl - constrains: - - liblapack 3.11.0 5*_mkl - - liblapacke 3.11.0 5*_mkl - - blas 2.305 mkl - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 68079 - timestamp: 1765819124349 - conda: https://conda.anaconda.org/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2 sha256: fd1d153962764433fe6233f34a72cdeed5dcf8a883a85769e8295ce940b5b0c5 md5: c965a5aa0d5c1c37ffc62dff36e28400 @@ -3552,16 +3385,6 @@ packages: purls: [] size: 462942 timestamp: 1767821743793 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-21.1.8-h55c6f16_2.conda - sha256: 5fbeb2fc2673f0455af6079abf93faaf27f11a92574ad51565fa1ecac9a4e2aa - md5: 4cb5878bdb9ebfa65b7cdff5445087c5 - depends: - - __osx >=11.0 - license: Apache-2.0 WITH LLVM-exception - license_family: Apache - purls: [] - size: 570068 - timestamp: 1770238262922 - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda sha256: d789471216e7aba3c184cd054ed61ce3f6dac6f87a50ec69291b9297f8c18724 md5: c277e0a4d549b03ac1e9d6cbbe3d017b @@ -3682,19 +3505,6 @@ packages: purls: [] size: 1040478 timestamp: 1770252533873 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgcc-15.2.0-hcbb3090_17.conda - sha256: 07ba27f2ef1ce444ce5c99d0f9590772fc5b58ba73c993477bfad74b17dfaa79 - md5: 65c07cee234440ae4d5d340fc4b2e69a - depends: - - _openmp_mutex - constrains: - - libgomp 15.2.0 17 - - libgcc-ng ==15.2.0=*_17 - license: GPL-3.0-only WITH GCC-exception-3.1 - license_family: GPL - purls: [] - size: 402928 - timestamp: 1770254186829 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_17.conda sha256: bdfe50501e4a2d904a5eae65a7ae26e2b7a29b473ab084ad55d96080b966502e md5: 1478bfa85224a65ab096d69ffd2af1e5 @@ -3717,18 +3527,6 @@ packages: purls: [] size: 27515 timestamp: 1770252591906 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran-15.2.0-h07b0088_17.conda - sha256: 7b96f428cb932df8d7c1aa4e433ed29b779dd9571934afdf4f9093a85155a142 - md5: 45ba22eb5381fb602a45233d89ba27ae - depends: - - libgfortran5 15.2.0 hdae7583_17 - constrains: - - libgfortran-ng ==15.2.0=*_17 - license: GPL-3.0-only WITH GCC-exception-3.1 - license_family: GPL - purls: [] - size: 139757 - timestamp: 1770254394473 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-15.2.0-h68bc16d_17.conda sha256: b1c77b85da9a3e204de986f59e262268805c6a35dffdf3953f1b98407db2aef3 md5: 202fdf8cad9eea704c2b0d823d1732bf @@ -3742,18 +3540,6 @@ packages: purls: [] size: 2480824 timestamp: 1770252563579 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libgfortran5-15.2.0-hdae7583_17.conda - sha256: 9c41ff08f61c953cee13fc3df3c6245741e5a71e453b2c094a6d55b0eeda3669 - md5: c6329d871fb3207e9657c384128f5488 - depends: - - libgcc >=15.2.0 - constrains: - - libgfortran 15.2.0 - license: GPL-3.0-only WITH GCC-exception-3.1 - license_family: GPL - purls: [] - size: 599374 - timestamp: 1770254196706 - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-15.2.0-he0feb66_17.conda sha256: b961b5dd9761907a7179678b58a69bb4fc16b940eb477f635aea3aec0a3f17a6 md5: 51b78c6a757575c0d12f4401ffc67029 @@ -3824,21 +3610,6 @@ packages: purls: [] size: 8349777 timestamp: 1761058442526 -- conda: https://conda.anaconda.org/conda-forge/win-64/libhwloc-2.12.2-default_h4379cf1_1000.conda - sha256: 8cdf11333a81085468d9aa536ebb155abd74adc293576f6013fc0c85a7a90da3 - md5: 3b576f6860f838f950c570f4433b086e - depends: - - libwinpthread >=12.0.0.r4.gg4f2fc60ca - - libxml2 - - libxml2-16 >=2.14.6 - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 2411241 - timestamp: 1765104337762 - conda: https://conda.anaconda.org/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda sha256: c467851a7312765447155e071752d7bf9bf44d610a5687e32706f480aad2833f md5: 915f5995e94f60e9a4826e0b0920ee88 @@ -3849,32 +3620,6 @@ packages: purls: [] size: 790176 timestamp: 1754908768807 -- conda: https://conda.anaconda.org/conda-forge/win-64/libiconv-1.18-hc1393d2_2.conda - sha256: 0dcdb1a5f01863ac4e8ba006a8b0dc1a02d2221ec3319b5915a1863254d7efa7 - md5: 64571d1dd6cdcfa25d0664a5950fdaa2 - depends: - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: LGPL-2.1-only - purls: [] - size: 696926 - timestamp: 1754909290005 -- conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-5_h47877c9_openblas.conda - build_number: 5 - sha256: c723b6599fcd4c6c75dee728359ef418307280fa3e2ee376e14e85e5bbdda053 - md5: b38076eb5c8e40d0106beda6f95d7609 - depends: - - libblas 3.11.0 5_h4a7cf45_openblas - constrains: - - blas 2.305 openblas - - liblapacke 3.11.0 5*_openblas - - libcblas 3.11.0 5*_openblas - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 18200 - timestamp: 1765818857876 - conda: https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.11.0-7_h8876d29_netlib.conda build_number: 7 sha256: 4de5b6aef4b2d42b4f71c6a3673118f99e323aed2ba2a66a3ed435b574010b1e @@ -3893,36 +3638,6 @@ packages: purls: [] size: 2901209 timestamp: 1763440547062 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblapack-3.11.0-5_hd9741b5_openblas.conda - build_number: 5 - sha256: 735a6e6f7d7da6f718b6690b7c0a8ae4815afb89138aa5793abe78128e951dbb - md5: ca9d752201b7fa1225bca036ee300f2b - depends: - - libblas 3.11.0 5_h51639a9_openblas - constrains: - - libcblas 3.11.0 5*_openblas - - blas 2.305 openblas - - liblapacke 3.11.0 5*_openblas - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 18551 - timestamp: 1765819121855 -- conda: https://conda.anaconda.org/conda-forge/win-64/liblapack-3.11.0-5_hf9ab0e9_mkl.conda - build_number: 5 - sha256: a2d33f5cc2b8a9042f2af6981c6733ab1a661463823eaa56595a9c58c0ab77e1 - md5: e62c42a4196dee97d20400612afcb2b1 - depends: - - libblas 3.11.0 5_hf2e6a31_mkl - constrains: - - libcblas 3.11.0 5*_mkl - - blas 2.305 mkl - - liblapacke 3.11.0 5*_mkl - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 80225 - timestamp: 1765819148014 - conda: https://conda.anaconda.org/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda sha256: 755c55ebab181d678c12e49cced893598f2bab22d582fbbf4d8b83c18be207eb md5: c7c83eecbb72d88b940c249af56c8b17 @@ -3987,21 +3702,6 @@ packages: purls: [] size: 33731 timestamp: 1750274110928 -- conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.30-pthreads_h94d23a6_4.conda - sha256: 199d79c237afb0d4780ccd2fbf829cea80743df60df4705202558675e07dd2c5 - md5: be43915efc66345cccb3c310b6ed0374 - depends: - - __glibc >=2.17,<3.0.a0 - - libgcc >=14 - - libgfortran - - libgfortran5 >=14.3.0 - constrains: - - openblas >=0.3.30,<0.3.31.0a0 - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 5927939 - timestamp: 1763114673331 - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.31-pthreads_h94d23a6_0.conda sha256: 166217a610185f9e22b3f4e0f80174d81240d6cfac8026b2f0158ff4f32b289a md5: 97ad7535866bf922275706c519b5c21d @@ -4017,21 +3717,6 @@ packages: purls: [] size: 5937816 timestamp: 1768555660623 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libopenblas-0.3.30-openmp_ha158390_4.conda - sha256: ebbbc089b70bcde87c4121a083c724330f02a690fb9d7c6cd18c30f1b12504fa - md5: a6f6d3a31bb29e48d37ce65de54e2df0 - depends: - - __osx >=11.0 - - libgfortran - - libgfortran5 >=14.3.0 - - llvm-openmp >=19.1.7 - constrains: - - openblas >=0.3.30,<0.3.31.0a0 - license: BSD-3-Clause - license_family: BSD - purls: [] - size: 4284132 - timestamp: 1768547079205 - conda: https://conda.anaconda.org/conda-forge/linux-64/libopentelemetry-cpp-1.21.0-hb9b0907_1.conda sha256: ba9b09066f9abae9b4c98ffedef444bbbf4c068a094f6c77d70ef6f006574563 md5: 1c0320794855f457dea27d35c4c71e23 @@ -4234,18 +3919,6 @@ packages: purls: [] size: 40311 timestamp: 1766271528534 -- conda: https://conda.anaconda.org/conda-forge/win-64/libwinpthread-12.0.0.r4.gg4f2fc60ca-h57928b3_10.conda - sha256: 0fccf2d17026255b6e10ace1f191d0a2a18f2d65088fd02430be17c701f8ffe0 - md5: 8a86073cf3b343b87d03f41790d8b4e5 - depends: - - ucrt - constrains: - - pthreads-win32 <0.0a0 - - msys2-conda-epoch <0.0a0 - license: MIT AND BSD-3-Clause-Clear - purls: [] - size: 36621 - timestamp: 1759768399557 - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda sha256: 6ae68e0b86423ef188196fff6207ed0c8195dd84273cb5623b85aa08033a410c md5: 5aa797f8787fe7a17d1b0821485b5adc @@ -4270,41 +3943,6 @@ packages: purls: [] size: 697033 timestamp: 1761766011241 -- conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-2.15.1-h779ef1b_1.conda - sha256: 8b47d5fb00a6ccc0f495d16787ab5f37a434d51965584d6000966252efecf56d - md5: 68dc154b8d415176c07b6995bd3a65d9 - depends: - - icu >=78.1,<79.0a0 - - libiconv >=1.18,<2.0a0 - - liblzma >=5.8.1,<6.0a0 - - libxml2-16 2.15.1 h3cfd58e_1 - - libzlib >=1.3.1,<2.0a0 - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: MIT - license_family: MIT - purls: [] - size: 43387 - timestamp: 1766327259710 -- conda: https://conda.anaconda.org/conda-forge/win-64/libxml2-16-2.15.1-h3cfd58e_1.conda - sha256: a857e941156b7f462063e34e086d212c6ccbc1521ebdf75b9ed66bd90add57dc - md5: 07d73826fde28e7dbaec52a3297d7d26 - depends: - - icu >=78.1,<79.0a0 - - libiconv >=1.18,<2.0a0 - - liblzma >=5.8.1,<6.0a0 - - libzlib >=1.3.1,<2.0a0 - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - constrains: - - libxml2 2.15.1 - license: MIT - license_family: MIT - purls: [] - size: 518964 - timestamp: 1766327232819 - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda sha256: d4bfe88d7cb447768e31650f06257995601f89076080e76df55e3112d4e47dc4 md5: edb0dca6bc32e4f4789199455a1dbeb8 @@ -4344,34 +3982,6 @@ packages: purls: [] size: 55476 timestamp: 1727963768015 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/llvm-openmp-21.1.8-h4a912ad_0.conda - sha256: 56bcd20a0a44ddd143b6ce605700fdf876bcf5c509adc50bf27e76673407a070 - md5: 206ad2df1b5550526e386087bef543c7 - depends: - - __osx >=11.0 - constrains: - - openmp 21.1.8|21.1.8.* - - intel-openmp <0.0a0 - license: Apache-2.0 WITH LLVM-exception - license_family: APACHE - purls: [] - size: 285974 - timestamp: 1765964756583 -- conda: https://conda.anaconda.org/conda-forge/win-64/llvm-openmp-21.1.8-h4fa8253_0.conda - sha256: 145c4370abe870f10987efa9fc15a8383f1dab09abbc9ad4ff15a55d45658f7b - md5: 0d8b425ac862bcf17e4b28802c9351cb - depends: - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - constrains: - - intel-openmp <0.0a0 - - openmp 21.1.8|21.1.8.* - license: Apache-2.0 WITH LLVM-exception - license_family: APACHE - purls: [] - size: 347566 - timestamp: 1765964942856 - conda: https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda sha256: 47326f811392a5fd3055f0f773036c392d26fdb32e4d8e7a8197eed951489346 md5: 9de5350a85c4a20c685259b889aa6393 @@ -4571,20 +4181,6 @@ packages: - pkg:pypi/mistune?source=hash-mapping size: 74250 timestamp: 1766504456031 -- conda: https://conda.anaconda.org/conda-forge/win-64/mkl-2025.3.0-hac47afa_455.conda - sha256: b2b4c84b95210760e4d12319416c60ab66e03674ccdcbd14aeb59f82ebb1318d - md5: fd05d1e894497b012d05a804232254ed - depends: - - llvm-openmp >=21.1.8 - - tbb >=2022.3.0 - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: LicenseRef-IntelSimplifiedSoftwareOct2022 - license_family: Proprietary - purls: [] - size: 100224829 - timestamp: 1767634557029 - pypi: https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl name: mpmath version: 1.3.0 @@ -4882,6 +4478,21 @@ packages: requires_dist: - numpy>=1.23.0 requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/1b/46/6fa4ea94f1ddf969b2ee941290cca6f1bfac92b53c76ae5f44afe17ceb69/numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: numpy + version: 2.4.2 + sha256: c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl + name: numpy + version: 2.4.2 + sha256: 7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1 + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl + name: numpy + version: 2.4.2 + sha256: b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695 + requires_python: '>=3.11' - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-1.26.4-py311h64a7726_0.conda sha256: 3f4365e11b28e244c95ba8579942b0802761ba7bb31c026f50d1a9ea9c728149 md5: a502d7aad449a1206efb366d6a12c52d @@ -4901,66 +4512,6 @@ packages: - pkg:pypi/numpy?source=hash-mapping size: 8065890 timestamp: 1707225944355 -- conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.4.2-py311h2e04523_1.conda - sha256: 2f9971a62316b9acb6ade749cebb59ffe750d1c2d99fe7061c6440589f6d3299 - md5: a8105076864776eceae69d64d30e24d7 - depends: - - python - - __glibc >=2.17,<3.0.a0 - - libstdcxx >=14 - - libgcc >=14 - - libblas >=3.9.0,<4.0a0 - - python_abi 3.11.* *_cp311 - - libcblas >=3.9.0,<4.0a0 - - liblapack >=3.9.0,<4.0a0 - constrains: - - numpy-base <0a0 - license: BSD-3-Clause - license_family: BSD - purls: - - pkg:pypi/numpy?source=compressed-mapping - size: 9385101 - timestamp: 1770098496391 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/numpy-2.4.2-py311had1e860_1.conda - sha256: 09a06de7adea145124618b023e5b0da2949a7211083d0805c21960ab980e053b - md5: bebff6d1b28a10a57a586cc449688324 - depends: - - python - - __osx >=11.0 - - python 3.11.* *_cpython - - libcxx >=19 - - libblas >=3.9.0,<4.0a0 - - python_abi 3.11.* *_cp311 - - libcblas >=3.9.0,<4.0a0 - - liblapack >=3.9.0,<4.0a0 - constrains: - - numpy-base <0a0 - license: BSD-3-Clause - license_family: BSD - purls: - - pkg:pypi/numpy?source=hash-mapping - size: 7451944 - timestamp: 1770098395802 -- conda: https://conda.anaconda.org/conda-forge/win-64/numpy-2.4.2-py311h80b3fa1_1.conda - sha256: c5cd26fb28d92d6c3843b96489f433ef87d1866d03a746f7228230b74bef431a - md5: a824c6667179120c458beb9e9394932f - depends: - - python - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - - ucrt >=10.0.20348.0 - - python_abi 3.11.* *_cp311 - - libcblas >=3.9.0,<4.0a0 - - liblapack >=3.9.0,<4.0a0 - - libblas >=3.9.0,<4.0a0 - constrains: - - numpy-base <0a0 - license: BSD-3-Clause - license_family: BSD - purls: - - pkg:pypi/numpy?source=hash-mapping - size: 7803678 - timestamp: 1770098404597 - pypi: https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl name: nvidia-cublas-cu12 version: 12.8.4.1 @@ -6628,6 +6179,138 @@ packages: - safetensors[testing] ; extra == 'all' - safetensors[all] ; extra == 'dev' requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/52/c8/08629657ac6c0da198487ce8cd3de78e02cfde42b7f34117d56a3fe249dc/scipy-1.17.0-cp311-cp311-win_amd64.whl + name: scipy + version: 1.17.0 + sha256: 255c0da161bd7b32a6c898e7891509e8a9289f0b1c6c7d96142ee0d2b114c2ea + requires_dist: + - numpy>=1.26.4,<2.7 + - pytest>=8.0.0 ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-xdist ; extra == 'test' + - asv ; extra == 'test' + - mpmath ; extra == 'test' + - gmpy2 ; extra == 'test' + - threadpoolctl ; extra == 'test' + - scikit-umfpack ; extra == 'test' + - pooch ; extra == 'test' + - hypothesis>=6.30 ; extra == 'test' + - array-api-strict>=2.3.1 ; extra == 'test' + - cython ; extra == 'test' + - meson ; extra == 'test' + - ninja ; sys_platform != 'emscripten' and extra == 'test' + - sphinx>=5.0.0,<8.2.0 ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' + - sphinx-copybutton ; extra == 'doc' + - sphinx-design>=0.4.0 ; extra == 'doc' + - matplotlib>=3.5 ; extra == 'doc' + - numpydoc ; extra == 'doc' + - jupytext ; extra == 'doc' + - myst-nb>=1.2.0 ; extra == 'doc' + - pooch ; extra == 'doc' + - jupyterlite-sphinx>=0.19.1 ; extra == 'doc' + - jupyterlite-pyodide-kernel ; extra == 'doc' + - linkify-it-py ; extra == 'doc' + - tabulate ; extra == 'doc' + - click<8.3.0 ; extra == 'dev' + - spin ; extra == 'dev' + - mypy==1.10.0 ; extra == 'dev' + - typing-extensions ; extra == 'dev' + - types-psutil ; extra == 'dev' + - pycodestyle ; extra == 'dev' + - ruff>=0.12.0 ; extra == 'dev' + - cython-lint>=0.12.2 ; extra == 'dev' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/5e/5f/a6b38f79a07d74989224d5f11b55267714707582908a5f1ae854cf9a9b84/scipy-1.17.0-cp311-cp311-macosx_12_0_arm64.whl + name: scipy + version: 1.17.0 + sha256: ef28d815f4d2686503e5f4f00edc387ae58dfd7a2f42e348bb53359538f01558 + requires_dist: + - numpy>=1.26.4,<2.7 + - pytest>=8.0.0 ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-xdist ; extra == 'test' + - asv ; extra == 'test' + - mpmath ; extra == 'test' + - gmpy2 ; extra == 'test' + - threadpoolctl ; extra == 'test' + - scikit-umfpack ; extra == 'test' + - pooch ; extra == 'test' + - hypothesis>=6.30 ; extra == 'test' + - array-api-strict>=2.3.1 ; extra == 'test' + - cython ; extra == 'test' + - meson ; extra == 'test' + - ninja ; sys_platform != 'emscripten' and extra == 'test' + - sphinx>=5.0.0,<8.2.0 ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' + - sphinx-copybutton ; extra == 'doc' + - sphinx-design>=0.4.0 ; extra == 'doc' + - matplotlib>=3.5 ; extra == 'doc' + - numpydoc ; extra == 'doc' + - jupytext ; extra == 'doc' + - myst-nb>=1.2.0 ; extra == 'doc' + - pooch ; extra == 'doc' + - jupyterlite-sphinx>=0.19.1 ; extra == 'doc' + - jupyterlite-pyodide-kernel ; extra == 'doc' + - linkify-it-py ; extra == 'doc' + - tabulate ; extra == 'doc' + - click<8.3.0 ; extra == 'dev' + - spin ; extra == 'dev' + - mypy==1.10.0 ; extra == 'dev' + - typing-extensions ; extra == 'dev' + - types-psutil ; extra == 'dev' + - pycodestyle ; extra == 'dev' + - ruff>=0.12.0 ; extra == 'dev' + - cython-lint>=0.12.2 ; extra == 'dev' + requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/ef/df/df1457c4df3826e908879fe3d76bc5b6e60aae45f4ee42539512438cfd5d/scipy-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + name: scipy + version: 1.17.0 + sha256: dac97a27520d66c12a34fd90a4fe65f43766c18c0d6e1c0a80f114d2260080e4 + requires_dist: + - numpy>=1.26.4,<2.7 + - pytest>=8.0.0 ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-xdist ; extra == 'test' + - asv ; extra == 'test' + - mpmath ; extra == 'test' + - gmpy2 ; extra == 'test' + - threadpoolctl ; extra == 'test' + - scikit-umfpack ; extra == 'test' + - pooch ; extra == 'test' + - hypothesis>=6.30 ; extra == 'test' + - array-api-strict>=2.3.1 ; extra == 'test' + - cython ; extra == 'test' + - meson ; extra == 'test' + - ninja ; sys_platform != 'emscripten' and extra == 'test' + - sphinx>=5.0.0,<8.2.0 ; extra == 'doc' + - intersphinx-registry ; extra == 'doc' + - pydata-sphinx-theme>=0.15.2 ; extra == 'doc' + - sphinx-copybutton ; extra == 'doc' + - sphinx-design>=0.4.0 ; extra == 'doc' + - matplotlib>=3.5 ; extra == 'doc' + - numpydoc ; extra == 'doc' + - jupytext ; extra == 'doc' + - myst-nb>=1.2.0 ; extra == 'doc' + - pooch ; extra == 'doc' + - jupyterlite-sphinx>=0.19.1 ; extra == 'doc' + - jupyterlite-pyodide-kernel ; extra == 'doc' + - linkify-it-py ; extra == 'doc' + - tabulate ; extra == 'doc' + - click<8.3.0 ; extra == 'dev' + - spin ; extra == 'dev' + - mypy==1.10.0 ; extra == 'dev' + - typing-extensions ; extra == 'dev' + - types-psutil ; extra == 'dev' + - pycodestyle ; extra == 'dev' + - ruff>=0.12.0 ; extra == 'dev' + - cython-lint>=0.12.2 ; extra == 'dev' + requires_python: '>=3.11' - conda: https://conda.anaconda.org/conda-forge/linux-64/scipy-1.17.0-py311hbe70eeb_1.conda sha256: b9582e96d703b2f2f61efc7394c886aefa5ab44983818bfc4a1894afc099561c md5: f4dda6316cc4718cbcab7009b5d60c41 @@ -6651,50 +6334,6 @@ packages: - pkg:pypi/scipy?source=compressed-mapping size: 16967163 timestamp: 1768800888207 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/scipy-1.17.0-py311he9931d0_1.conda - sha256: d9f37c85cbf689be3672c8264eb81585ad8f6041a2fe545ec978f42e5da0202c - md5: 9c5c9dbdaf090ba8be3beb34c01495d0 - depends: - - __osx >=11.0 - - libblas >=3.9.0,<4.0a0 - - libcblas >=3.9.0,<4.0a0 - - libcxx >=19 - - libgfortran - - libgfortran5 >=14.3.0 - - liblapack >=3.9.0,<4.0a0 - - numpy <2.7 - - numpy >=1.23,<3 - - numpy >=1.25.2 - - python >=3.11,<3.12.0a0 - - python >=3.11,<3.12.0a0 *_cpython - - python_abi 3.11.* *_cp311 - license: BSD-3-Clause - license_family: BSD - purls: - - pkg:pypi/scipy?source=compressed-mapping - size: 14030449 - timestamp: 1768801949072 -- conda: https://conda.anaconda.org/conda-forge/win-64/scipy-1.17.0-py311h9c22a71_1.conda - sha256: c6896bbe8cb62b1743b86e4bae8c509233231412bf7ffd92bf0d5036a617dc8e - md5: 0d03c857517a5db3c1af5b553a528fac - depends: - - libblas >=3.9.0,<4.0a0 - - libcblas >=3.9.0,<4.0a0 - - liblapack >=3.9.0,<4.0a0 - - numpy <2.7 - - numpy >=1.23,<3 - - numpy >=1.25.2 - - python >=3.11,<3.12.0a0 - - python_abi 3.11.* *_cp311 - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: BSD-3-Clause - license_family: BSD - purls: - - pkg:pypi/scipy?source=hash-mapping - size: 14988880 - timestamp: 1768801728977 - conda: https://conda.anaconda.org/conda-forge/linux-64/scitokens-cpp-1.3.0-h096d96b_0.conda sha256: 11ad442837d2bd3c856c8a7ed08754ca430e6779999d898d1fa313fcd670458c md5: 946024dbdba971eeda33da76ae586694 @@ -6876,19 +6515,6 @@ packages: - blosc2>=2.3.0 - typing-extensions>=4.4.0 requires_python: '>=3.11' -- conda: https://conda.anaconda.org/conda-forge/win-64/tbb-2022.3.0-h3155e25_2.conda - sha256: abd9a489f059fba85c8ffa1abdaa4d515d6de6a3325238b8e81203b913cf65a9 - md5: 0f9817ffbe25f9e69ceba5ea70c52606 - depends: - - libhwloc >=2.12.2,<2.12.3.0a0 - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: Apache-2.0 - license_family: APACHE - purls: [] - size: 155869 - timestamp: 1767886839029 - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda sha256: 6b6727a13d1ca6a23de5e6686500d0669081a117736a87c8abf444d60c1e40eb md5: 17b43cee5cc84969529d5d0b0309b2cb diff --git a/pyproject.toml b/pyproject.toml index 464be28..17c0788 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,12 @@ dependencies = [ "matplotlib>=3.10.8,<4", "numpy>=1.26.4,<3", "pandas>=3.0.0,<4", + "scipy", "tables>=3.10.2,<4", "torch", "torchinfo>=1.8.0,<2", "torchvision", - "transformers>=5.1.0,<6" + "transformers>=5.1.0,<6", ] dynamic = ["version"] @@ -48,10 +49,7 @@ torchvision = { version = ">=0.20.1", index = "https://download.pytorch.org/whl/ [tool.pixi.dependencies] python = ">=3.11,<3.12" -omegaconf = ">=2.3.0,<3" hydra-core = ">=1.3.2,<2" -scipy = ">=1.17.0,<2" -debugpy = ">=1.8.20,<2" [tool.pixi.feature.fdp] platforms = ["linux-64"] diff --git a/scripts/data_fetching_omega/config_atlas.yaml b/scripts/data_fetching_omega/config_atlas.yaml index 4771c7b..26a6aaf 100644 --- a/scripts/data_fetching_omega/config_atlas.yaml +++ b/scripts/data_fetching_omega/config_atlas.yaml @@ -1652,6 +1652,65 @@ trees: - \AOT::TRIANGULARITY_U - \AOT::TRIANGULARITY_L - \AOT::Q + SPECTROSCOPY: + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:CIII_977 + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:CII_651 + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:CII_904 + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:CIV_1550 + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:DLYA_1215 + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:DLYB_1025 + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:INTENSITIES + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:INT_TIMES + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:START_TIMES + - \SPECTROSCOPY::TOP.DIVSPRED.RAW:WAVELENGTHS + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L01_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L02_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L03_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L04_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L05_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L06_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L07_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L08_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L09_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L10_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L11_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L12_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L13_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L14_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L15_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L16_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L17_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L18_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L19_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L20_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L21_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L22_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L23_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_L24_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U01_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U02_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U03_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U04_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U05_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U06_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U07_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U08_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U09_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U10_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U11_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U12_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U13_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U14_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U15_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U16_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U17_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U18_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U19_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U20_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U21_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U22_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U23_P + - \SPECTROSCOPY::TOP.PRAD.BOLOM.PRAD_01.POWER:BOL_U24_P ptdata: - MPI1A322D - MPI3A322D diff --git a/scripts/data_fetching_omega/merge_h5.py b/scripts/data_fetching_omega/merge_h5.py new file mode 100644 index 0000000..a4ef9c7 --- /dev/null +++ b/scripts/data_fetching_omega/merge_h5.py @@ -0,0 +1,183 @@ +""" +Add data from source H5 files to a target H5 file (same shot). + +Usage: + python merge_h5.py target.h5 source1.h5 source2.h5 ... + python merge_h5.py 200000.h5 200000_chiron.h5 200000_extra.h5 +""" + +import h5py +import sys +import argparse +from pathlib import Path + + +def add_to_h5( + target_file: str | Path, + source_files: list[str | Path], + strategy: str = 'skip', + verbose=True +): + """ + Add trees/signals from source files to target file. + + Parameters + ---------- + target_file: Path to target HDF5 file (modified in place) + source_files: List of source HDF5 files to add from + strategy: How to handle duplicates ('skip', 'overwrite', 'error') + verbose: Print progress messages + """ + if not Path(target_file).exists(): + print(f"Error: Target file does not exist: {target_file}") + print("Create it first or use one of the source files as target") + sys.exit(1) + + if verbose: + print(f"Target file: {target_file}") + + with h5py.File(target_file, 'a') as f_target: + stats = { + 'files_processed': 0, + 'trees_added': 0, + 'signals_added': 0, + 'signals_skipped': 0, + 'signals_overwritten': 0 + } + + for source_file in source_files: + if not Path(source_file).exists(): + print(f"Warning: {source_file} does not exist, skipping") + continue + + if Path(source_file).resolve() == Path(target_file).resolve(): + if verbose: + print(f"\nSkipping {source_file} (same as target)") + continue + + if verbose: + print(f"\nAdding from: {source_file}") + + try: + with h5py.File(source_file, 'r') as f_source: + # Iterate over shots + for shot_name in f_source.keys(): + if verbose: + print(f" Shot {shot_name}:") + + # Ensure shot exists in target + if shot_name not in f_target: + f_target.create_group(shot_name) + if verbose: + print(f" Created shot group") + + # Iterate over trees + for tree_name in f_source[shot_name].keys(): + tree_path = f"{shot_name}/{tree_name}" + + if tree_path not in f_target: + f_target.create_group(tree_path) + stats['trees_added'] += 1 + if verbose: + print(f" Tree {tree_name} (new)") + else: + if verbose: + print(f" Tree {tree_name} (existing)") + + # Iterate over signals + for signal_name in f_source[shot_name][tree_name].keys(): + signal_path = f"{shot_name}/{tree_name}/{signal_name}" + + # Check if signal exists + if signal_path in f_target: + if strategy == 'skip': + stats['signals_skipped'] += 1 + if verbose: + print(f" {signal_name} (skipped)") + continue + elif strategy == 'error': + raise ValueError( + f"Duplicate signal: {signal_path}") + elif strategy == 'overwrite': + del f_target[signal_path] + stats['signals_overwritten'] += 1 + if verbose: + print(f" {signal_name} (overwritten)") + + # Copy signal + f_source.copy(f_source[signal_path], f_target, + signal_path) + stats['signals_added'] += 1 + + if verbose and strategy != 'skip': + print(f" {signal_name} (added)") + + stats['files_processed'] += 1 + + except Exception as e: + print(f"Error processing {source_file}: {e}") + import traceback + traceback.print_exc() + continue + + # Print summary + if verbose: + print("\n" + "=" * 60) + print("Summary:") + print("=" * 60) + print(f"Files processed: {stats['files_processed']}") + print(f"Trees added: {stats['trees_added']}") + print(f"Signals added: {stats['signals_added']}") + if stats['signals_skipped'] > 0: + print(f"Signals skipped (duplicates): {stats['signals_skipped']}") + if stats['signals_overwritten'] > 0: + print(f"Signals overwritten: {stats['signals_overwritten']}") + print(f"\nTarget file updated: {target_file}") + + +def main(): + parser = argparse.ArgumentParser( + description='Add data from source H5 files to target H5 file', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog= + """Examples: + # Add chiron data to existing atlas file + python merge_h5.py 200000_atlas.h5 200000_chiron.h5 + + # Add multiple sources to target + python merge_h5.py 200000.h5 200000_extra1.h5 200000_extra2.h5 + + # Overwrite duplicates + python merge_h5.py 200000.h5 200000_new.h5 --strategy overwrite + + Workflow: + 1. Fetch atlas data -> 200000_atlas.h5 + 2. Fetch chiron data -> 200000_chiron.h5 + 3. Merge: python merge_h5.py 200000_atlas.h5 200000_chiron.h5 + 4. Result: 200000_atlas.h5 now contains both atlas and chiron trees + """ + ) + + parser.add_argument('target', help='Target HDF5 file (will be modified)') + parser.add_argument('sources', nargs='+', help='Source HDF5 files to add') + parser.add_argument( + '--strategy', choices=['skip', 'overwrite', 'error'], default='skip', + help='How to handle duplicate signals (default: skip)' + ) + parser.add_argument( + '-q', '--quiet', action='store_true', help='Suppress progress messages' + ) + + args = parser.parse_args() + + # Add data + add_to_h5( + args.target, + args.sources, + strategy=args.strategy, + verbose=not args.quiet + ) + + +if __name__ == '__main__': + main() diff --git a/scripts/data_preparation/prepare_data.py b/scripts/data_preparation/prepare_data.py new file mode 100644 index 0000000..ac9d979 --- /dev/null +++ b/scripts/data_preparation/prepare_data.py @@ -0,0 +1,721 @@ +import numpy as np +import h5py +import hydra +import logging +from multiprocessing import Pool +from functools import partial +from omegaconf import DictConfig, OmegaConf +from typing import Union +from pathlib import Path +from tqdm.auto import tqdm +from scipy.interpolate import interp1d +import warnings +import os + + +log = logging.getLogger(__name__) + + +class SignalLoader: + """Load grouped signals from MDSPlus HDF5 files.""" + + def __init__(self, h5_file_path: str | Path, verbose: bool = True): + """ + Initialize loader with HDF5 file path. + + Parameters + ---------- + h5_file_path : str | Path + Path to HDF5 file (e.g., '/path/to/200000.h5') + verbose : bool, default=True + Print warnings about missing signals. + """ + self.h5_file = h5py.File(h5_file_path, 'r') + self.shot_number = Path(h5_file_path).stem + self.verbose = verbose + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.h5_file.close() + + def load_signal_data( + self, + tree: str, + signal_path: str, + data_key: str = 'data', + time_key: str = 'dim0' + ) -> dict[str, np.ndarray]: + """ + Load a single signal's data and time arrays from HDF5. + + Parameters + ---------- + tree : str + Tree name (e.g., 'D3D', 'IRTV') + signal_path : str + Full signal path (e.g., '\\SPECTROSCOPY::FS02') + data_key : str + HDF5 dataset name for signal data (default: 'data') + time_key : str + HDF5 dataset name for time axis (default: 'dim0') + + Returns + ------- + Dictionary with 'data' and 'time' keys, or empty dict if not found + """ + try: + # Access: f[shot_number][tree][signal_path] + if self.shot_number not in self.h5_file: + if self.verbose: + warnings.warn(f"Shot {self.shot_number} not in HDF5 file") + return {} + + shot_group = self.h5_file[self.shot_number] + + if tree not in shot_group: + if self.verbose: + warnings.warn( + f"Tree '{tree}' not found for shot {self.shot_number}") + return {} + + tree_group = shot_group[tree] + + if signal_path not in tree_group: + if self.verbose: + warnings.warn( + f"Signal '{signal_path}' not found in tree '{tree}'") + return {} + + signal_group = tree_group[signal_path] + + # Load data and time + result = {} + + if data_key in signal_group: + result['data'] = signal_group[data_key][:] + else: + if self.verbose: + warnings.warn(f"Data key '{data_key}' not found " + f"in {tree}/{signal_path}") + result['data'] = np.array([]) + + if time_key in signal_group: + result['time'] = signal_group[time_key][:] + else: + # Time might not exist for all signals + result['time'] = np.array([]) + + return result + + except Exception as e: + if self.verbose: + warnings.warn(f"Error loading {tree}/{signal_path}: {e}") + return {} + + def load_signal_group( + self, + tree: str, + signal_paths: list[str], + data_key: str = 'data', + time_key: str = 'dim0' + ) -> dict[str, Union[np.ndarray, list[np.ndarray]]]: + """ + Load multiple signals from the same tree. + + Parameters + ---------- + tree : str + Tree name (e.g., 'D3D') + signal_paths : str + List of signal paths + data_key : str + HDF5 dataset name for signal data + time_key : str + HDF5 dataset name for time axis + + Returns + ------- + Dictionary with: + - 'data': Stacked array (channels x time) or list if shapes differ + - 'time': Time array or list of time arrays + - 'valid_indices': List of indices where data was successfully loaded + - 'num_valid': Number of valid signals + """ + data_list = [] + time_list = [] + valid_indices = [] + + for idx, path in enumerate(signal_paths): + signal_data = self.load_signal_data(tree, path, data_key, time_key) + + if signal_data and len(signal_data.get('data', [])) > 0: + data_list.append(signal_data['data']) + time_list.append(signal_data.get('time', np.array([]))) + valid_indices.append(idx) + else: + data_list.append(np.array([])) + time_list.append(np.array([])) + + if not data_list: + warnings.warn(f"No valid signals loaded from {len(signal_paths)} " + f"paths in tree {tree}") + return { + 'data': np.array([]), + 'time': np.array([]), + 'valid_indices': [], + 'num_valid': 0 + } + + # Check if we can stack the data + shapes = [d.shape for d in data_list] + all_same_shape = len(set(shapes)) == 1 + + result = { + 'valid_indices': valid_indices, + 'num_valid': len(valid_indices) + } + + if all_same_shape: + # Stack data into array (C x T) for 1D or (C x ...) + result['data'] = np.stack(data_list, axis=0) + if result['data'].shape[0] == 1: + result['data'] = result['data'].squeeze(0) + + # Check if all time arrays are the same + if time_list and len(time_list[0]) > 0: + time_shapes = [t.shape for t in time_list if len(t) > 0] + if len(set(time_shapes)) == 1: + result['time'] = time_list[0] # Use first time array + else: + # Stack different times + result['time'] = np.stack(time_list, axis=0) + else: + result['time'] = np.array([]) + else: + # Keep as list - shapes don't match + result['data'] = data_list + result['time'] = time_list + + if self.verbose: + warnings.warn(f"Signals have mismatched shapes: {shapes}") + + return result + + def load_from_config(self, config: dict) -> dict[str, dict]: + """ + Load all signal groups from a processing config. + + Parameters + ---------- + config : dict + Dictionary with 'signals' key containing groups + + Returns + ------- + Dictionary mapping group names to loaded data + """ + results = {} + + if 'signals' not in config: + raise ValueError("Config must have 'signals' key") + + for group_name, group_config in config['signals'].items(): + if self.verbose: + print(f"\nLoading group: {group_name}") + + tree = group_config['tree'] + signal_paths = group_config['input_key'] + data_key = group_config.get('input_ykey', 'data') # ykey is data + time_key = group_config.get('input_xkey', 'dim0') # xkey is time + + # Load signals + loaded = self.load_signal_group( + tree=tree, + signal_paths=signal_paths, + data_key=data_key, + time_key=time_key + ) + + # Add config metadata + loaded['config'] = group_config + loaded['tree'] = tree + + results[group_name] = loaded + + # Print summary + if (isinstance(loaded['data'], np.ndarray) + and loaded['data'].size > 0): + print(f"Loaded {loaded['num_valid']}/" + f"{len(signal_paths)} channels") + print(f" Data shape: {loaded['data'].shape}") + if (isinstance(loaded['time'], np.ndarray) + and len(loaded['time']) > 0): + print(f" Time shape: {loaded['time'].shape}") + if loaded['time'].ndim == 1: + print(f" Time range: {loaded['time'][0]:.3f} to " + f"{loaded['time'][-1]:.3f} s") + elif isinstance(loaded['data'], list) and len(loaded['data']) > 0: + print(f" Loaded {len(loaded['data'])}/{len(signal_paths)}" + f" signals (unstacked)") + print( + f" Shapes: {[d.shape for d in loaded['data'][:3]]}...") + else: + print(f" No valid data loaded") + + return results + + +def process_shot( + h5_file: str | Path, + config: dict, + verbose: bool = True +) -> dict[str, dict]: + """ + Load shot data from HDF5 using config. + + Parameters + ---------- + h5_file : str | Path + Path to HDF5 file + config : dict + Loaded config dictionary + verbose : bool + Print progress + + Returns + ------- + Dictionary of loaded signal groups + """ + with SignalLoader(h5_file, verbose=verbose) as loader: + data = loader.load_from_config(config) + + return data + + +def _resample_time_series(data, time, target_frequency): + """ + Resample non-uniformly sampled time series to uniform sampling. + + Parameters: + ----------- + data : np.ndarray, shape (n_samples, ...) + Time series data + time : np.ndarray, shape (n_samples,) + Time axis (can be non-uniform) + target_frequency : float + Desired sampling frequency in Hz + + Returns: + -------- + resampled_data : np.ndarray + Uniformly resampled data + new_time : np.ndarray + New uniform time axis + """ + if len(data) <= 1: + return (np.asarray(time, dtype=float).copy(), + np.asarray(data, dtype=float).copy()) + + # Calculate target sampling period + dt = 1.0 / target_frequency + + # Create uniform time grid + n_samples = int(np.ceil((time[-1] - time[0]) / dt)) + 1 + new_time = time[0] + np.arange(n_samples) * dt + + # Handle multi-dimensional data + original_shape = data.shape + if data.ndim > 1: + # Flatten all dimensions except the first (time) + data_flat = data.reshape(data.shape[0], -1) + resampled_flat = np.full((len(new_time), data_flat.shape[1]), np.nan) + + # Interpolate each channel, handling NaNs + for i in range(data_flat.shape[1]): + # Find valid (non-NaN) data points + valid_mask = ~np.isnan(data_flat[:, i]) + + if np.sum(valid_mask) >= 2: # Need at least 2 points + valid_time = time[valid_mask] + valid_data = data_flat[valid_mask, i] + + # Only interpolate within the range of valid data + interpolator = interp1d(valid_time, valid_data, kind='linear', + bounds_error=False, fill_value=np.nan) + resampled_flat[:, i] = interpolator(new_time) + + # Reshape back to original dimensions (except time axis) + new_shape = (len(new_time),) + original_shape[1:] + resampled_data = resampled_flat.reshape(new_shape) + else: + # 1D case + valid_mask = ~np.isnan(data) + + if np.sum(valid_mask) >= 2: + valid_time = time[valid_mask] + valid_data = data[valid_mask] + + interpolator = interp1d(valid_time, valid_data, kind='linear', + bounds_error=False, fill_value=np.nan) + resampled_data = interpolator(new_time) + else: + # Not enough valid data to interpolate + resampled_data = np.full(len(new_time), np.nan) + + return new_time, resampled_data + + +def resample_signal_groups(loaded_data: dict[str, dict]) -> dict[str, dict]: + """ + Resample all signal groups to their target sampling frequencies. + + All signals within a group are resampled to the SAME time grid. + + Parameters + ---------- + loaded_data : dict + Dictionary from process_shot() containing signal groups + + Returns + ------- + Dictionary with resampled data, same structure as input + """ + resampled = {} + + for group_name, group_data in loaded_data.items(): + print(f"\nResampling group: {group_name}") + + data = group_data['data'] + time = group_data['time'] + target_freq = group_data['config']['sampling_rate'] + num_channels = group_data['config']['num_channels'] + + # Skip if no valid data + if isinstance(data, np.ndarray) and data.size == 0: + print(f" Skipping - no data") + resampled[group_name] = group_data.copy() + continue + + # Handle stacked array (channels x time) - all share same time axis + if isinstance(data, np.ndarray) and time.ndim == 1: + if time.size == 0: + print(f" Skipping - no time axis") + resampled[group_name] = group_data.copy() + continue + + # Transpose from (channels, time) to (time, channels) + data_transposed = data.T + time = time / 1000 + + print(f" Data shape: {data.shape}") + print(f" Time range: {time[0]:.3f} to {time[-1]:.3f} s") + print(f" Target frequency: {target_freq} Hz") + + # Resample all channels together (they share time axis) + new_time, resampled_data = _resample_time_series( + data_transposed, time, target_freq + ) + + # Transpose back to (channels, time) + resampled_data = resampled_data.T + + print(f" Resampled: {resampled_data.shape}") + print(f" New time range: {new_time[0]:.3f} " + f"to {new_time[-1]:.3f} s") + + new_time = new_time * 1000 + + resampled[group_name] = group_data.copy() + resampled[group_name]['data'] = resampled_data + resampled[group_name]['time'] = new_time + + # Handle list of arrays OR stacked with different time axes + else: + print(f" Processing {len(data)} signals " + f"with potentially different time axes") + + # Step 1: Find global time range across ALL signals + # time_list = time if isinstance(time, list) else [time] * len(data) + time_list = time if isinstance(time, list) else list(time) + data_list = data if isinstance(data, list) else list(data) + + t_min = np.inf + t_max = -np.inf + + for t in time_list: + if isinstance(t, np.ndarray) and len(t) > 0: + t_min = min(t_min, t[0] / 1000) + t_max = max(t_max, t[-1] / 1000) + + if np.isinf(t_min) or np.isinf(t_max): + print(f" No valid time data found") + resampled[group_name] = group_data.copy() + continue + + # Step 2: Create single uniform time grid for entire group + dt = 1.0 / target_freq + n_samples = int(np.ceil((t_max - t_min) / dt)) + 1 + common_time = t_min + np.arange(n_samples) * dt + + print(f" Global time range: {t_min:.3f} to {t_max:.3f} s") + print(f" Common time grid: {len(common_time)} " + f"samples @ {target_freq} Hz") + common_time = common_time * 1000 + + # Step 3: Resample each signal to the COMMON time grid + # Detect spatial dimensions from the first non-empty multi-dim channel. + # For video the shape is (W, H, T) so spatial_shape = (W, H); + # for 1D time series spatial_shape stays None. + spatial_shape = None + for d in data_list: + if (isinstance(d, np.ndarray) and d.ndim > 1 + and d.size > 0): + spatial_shape = d.shape[:-1] # all axes except last (time) + break + + if spatial_shape is not None: + resampled_data_array = np.full( + (num_channels,) + spatial_shape + (len(common_time),), + np.nan, dtype='f8') + else: + resampled_data_array = np.full( + (num_channels, len(common_time)), np.nan, dtype='f8') + + for i, (signal_data, signal_time) in enumerate( + zip(data_list, time_list)): + if i >= num_channels: + break + + if (not isinstance(signal_data, np.ndarray) + or signal_data.size == 0): + continue # Leave as NaN + + if (not isinstance(signal_time, np.ndarray) + or signal_time.size == 0): + continue # Leave as NaN + + if signal_data.ndim == 1: + # 1D time series: interpolate directly + valid_mask = ~np.isnan(signal_data) + if np.sum(valid_mask) >= 2: + interpolator = interp1d( + signal_time[valid_mask], + signal_data[valid_mask], + kind='linear', + bounds_error=False, + fill_value=np.nan + ) + resampled_data_array[i, :] = interpolator(common_time) + else: + # Multi-dim channel (e.g. video shape (W, H, T)): + # time is the last axis; interpolate per spatial location. + ch_spatial = signal_data.shape[:-1] + n_time = signal_data.shape[-1] + + # (spatial..., T) -> (T, spatial_flat) + data_t = np.moveaxis(signal_data, -1, 0) + data_flat = data_t.reshape(n_time, -1) + + resampled_flat = np.full( + (len(common_time), data_flat.shape[1]), + np.nan, dtype='f8') + + for j in range(data_flat.shape[1]): + pixel_series = data_flat[:, j] + valid_mask = ~np.isnan(pixel_series) + if np.sum(valid_mask) >= 2: + interpolator = interp1d( + signal_time[valid_mask], + pixel_series[valid_mask], + kind='linear', + bounds_error=False, + fill_value=np.nan + ) + resampled_flat[:, j] = interpolator(common_time) + + # (new_T, spatial_flat) -> (spatial..., new_T) + resampled_nd = resampled_flat.reshape( + (len(common_time),) + ch_spatial) + resampled_data_array[i] = np.moveaxis(resampled_nd, 0, -1) + + valid_samples = int(np.sum(~np.isnan(resampled_data_array[i]))) + print(f" Channel {i}: {valid_samples} valid samples") + + resampled[group_name] = group_data.copy() + resampled[group_name]['data'] = resampled_data_array + resampled[group_name]['time'] = common_time / 1000. + print( + f" Resampled to common grid: {resampled_data_array.shape}") + + return resampled + + +def write_resampled_data( + resampled_data: dict[str, dict], + output_file: str | Path, +) -> None: + """ + Write resampled data to HDF5 file. + + Parameters + ---------- + resampled_data : dict + Dictionary from resample_signal_groups() + output_file : str | Path + Path to output HDF5 file + """ + print(f"\nWriting to {output_file}") + + with h5py.File(output_file, "w") as f: + for group_name, group_data in resampled_data.items(): + data = group_data['data'] + time = group_data['time'] + num_channels = group_data['config']['num_channels'] + + # Create HDF5 group for this signal group + grp = f.create_group(group_name) + + # Handle stacked arrays + if isinstance(data, np.ndarray): + # If data is empty, create NaN array with expected shape + if data.size == 0 or time.size == 0: + # Create minimal time axis (single point) + time_out = np.array([0.0]) + data_out = np.full((num_channels, 1), np.nan, dtype='f8') + print(f" ! {group_name}: " + f"No data, writing NaN array {data_out.shape}") + else: + time_out = time + + # Check if we have fewer channels than expected + if data.shape[0] < num_channels: + # Pad with NaN channels + missing_channels = num_channels - data.shape[0] + nan_channels = np.full( + (missing_channels, data.shape[1]), + np.nan, + dtype='f8') + data_out = np.vstack([data, nan_channels]) + print(f" ! {group_name}: " + f"Padded {missing_channels} NaN channels") + elif data.shape[0] > num_channels: + # Truncate extra channels (shouldn't happen) + data_out = data[:num_channels] + print(f" ! {group_name}: " + f"Truncated to {num_channels} channels") + else: + data_out = data + + grp.create_dataset('xdata', data=time_out, dtype='f8') + grp.create_dataset('ydata', data=data_out, dtype='f8') + + print(f" {group_name}: " + f"{data_out.shape} @ {len(time_out)} samples") + + # Handle list of arrays + elif isinstance(data, list): + # Find the longest time axis to use as reference + max_time_len = 1 + reference_time = np.array([0.0]) + + for t in time: + if isinstance(t, np.ndarray) and len(t) > max_time_len: + max_time_len = len(t) + reference_time = t + + # Build full data array with NaN padding + data_out = np.full( + (num_channels, max_time_len), np.nan, dtype='f8') + + for i, channel_data in enumerate(data): + if i >= num_channels: + break # Don't exceed expected channels + + if channel_data.size > 0: + # Copy available data + n_samples = min(len(channel_data), max_time_len) + data_out[i, :n_samples] = channel_data[:n_samples] + + grp.create_dataset('xdata', data=reference_time, dtype='f8') + grp.create_dataset('ydata', data=data_out, dtype='f8') + + print(f" {group_name}: {data_out.shape} " + f"@ {len(reference_time)} samples (from list)") + + # Set file permissions + os.chmod(output_file, 0o664) + + +def process_and_write_shot(shot: int, cfg_dict: dict) -> str | None: + """Worker function executed in a child process. + + Args: + shot: Shot number. + cfg_dict: Plain dict (not DictConfig – must be picklable). + + Returns: + None on success, or an error message string on failure. + """ + try: + input_data_path = Path(cfg_dict["input_data_path"]) + output_data_path = Path(cfg_dict["output_data_path"]) + output_data_path.mkdir(parents=True, exist_ok=True) + + input_file = input_data_path / f"{shot}.h5" + output_file = output_data_path / f"{shot}_processed.h5" + + data = process_shot(str(input_file), cfg_dict, verbose=True) + + resampled_data = resample_signal_groups(data) + + write_resampled_data(resampled_data, output_file) + + return None # success + + except Exception as e: + log.info(f"shot {shot}: {type(e).__name__}: {e}") + return f"shot {shot}: {type(e).__name__}: {e}" + + +@hydra.main(version_base=None, + config_path="../../src/tokamak_foundation_model/data/config", + config_name="config") +def main(cfg: DictConfig) -> None: + log.info(f"Config:\n{OmegaConf.to_yaml(cfg)}") + + mod_cfg = cfg.modalities + num_workers = mod_cfg.get("num_workers", 8) + + # ── filter to shots that exist in both paths ── + shots = list(cfg.shot_list.shots) + + if not shots: + log.error("No valid shots found – exiting.") + return + + # Convert to plain dict so it's picklable for multiprocessing + cfg_dict = OmegaConf.to_container(mod_cfg, resolve=True) + + log.info(f"Processing {len(shots)} shots with {num_workers} workers") + + worker = partial(process_and_write_shot, cfg_dict=cfg_dict) + + errors = [] + + with Pool(processes=num_workers) as pool: + for i, err in enumerate( + tqdm(pool.imap_unordered(worker, shots), total=len(shots))): + if err is not None: + log.error(err) + errors.append(err) + + log.info( + f"Done. {len(shots) - len(errors)}/{len(shots)} succeeded, " + f"{len(errors)} failed." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/slurm/benchmark_data_loader.sh b/scripts/slurm/benchmark_data_loader.sh new file mode 100755 index 0000000..8a9d5ff --- /dev/null +++ b/scripts/slurm/benchmark_data_loader.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#SBATCH --job-name=benchmark_data_loader +#SBATCH --output=logs/benchmark_data_loader.out +#SBATCH --error=logs/benchmark_data_loader.err +#SBATCH --cpus-per-task=32 +#SBATCH --nodes=1 +#SBATCH --mem-per-cpu=16G +#SBATCH --time=04:00:00 +#SBATCH --mail-type=all +#SBATCH --mail-user=ps9551@princeton.edu + +pixi run python ../training/benchmark_data_loader.py diff --git a/scripts/slurm/make_processing_stats.sh b/scripts/slurm/make_processing_stats.sh new file mode 100755 index 0000000..551164d --- /dev/null +++ b/scripts/slurm/make_processing_stats.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#SBATCH --job-name=make_processing_stats +#SBATCH --output=logs/make_processing_stats.out +#SBATCH --error=logs/make_processing_stats.err +#SBATCH --cpus-per-task=32 +#SBATCH --nodes=1 +#SBATCH --mem-per-cpu=16G +#SBATCH --time=02:00:00 +#SBATCH --mail-type=all +#SBATCH --mail-user=ps9551@princeton.edu + +pixi run python ../data_preparation/make_processing_stats.py diff --git a/scripts/slurm/prepare_data.sh b/scripts/slurm/prepare_data.sh new file mode 100755 index 0000000..1f1ac81 --- /dev/null +++ b/scripts/slurm/prepare_data.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#SBATCH --job-name=prepare_data # create a short name for your job +#SBATCH --output=logs/prepare_data.out +#SBATCH --error=logs/prepare_data.err +#SBATCH --cpus-per-task=32 # cpu-cores per task (>1 if multi-threaded tasks) +#SBATCH --nodes=1 # node count +#SBATCH --mem-per-cpu=16G # memory per cpu-core (4G is default) +#SBATCH --time=2:00:00 # total run time limit (HH:MM:SS) +#SBATCH --mail-type=all # send email on job start, end and fault +#SBATCH --mail-user=ps9551@princeton.edu + +pixi run python scripts/prepare_data.py diff --git a/scripts/training/benchmark_data_loader.py b/scripts/training/benchmark_data_loader.py new file mode 100644 index 0000000..fc07cdb --- /dev/null +++ b/scripts/training/benchmark_data_loader.py @@ -0,0 +1,44 @@ +from pathlib import Path +import torch +from torch.utils.data import ConcatDataset, DataLoader + +from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn +import time + + +def main(): + hdf5_files = sorted( + Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("[0-9]*_processed.h5") + ) + preprocessing_stats = torch.load("/scratch/gpfs/ps9551/FusionAIHub/scripts/slurm/preprocessing_stats.pt", weights_only=False) + + all_input_signals = [ + "mhr", "ece", "co2", "bes", # spectrograms + "gas", "ech", "pin", "tin", # actuators + "d_alpha", "mse", "ts_core_density", # diagnostics + "bolo", "irtv", "tangtv", # videos + # "text", # metadata + ] + + datasets = [ + TokamakH5Dataset( + hdf5_path=str(f), + input_signals=all_input_signals, + target_signals=all_input_signals, + preprocessing_stats=preprocessing_stats, + ) for f in hdf5_files] + combined = ConcatDataset(datasets) + dataloader = DataLoader(combined, batch_size=32, collate_fn=collate_fn, + num_workers=32) + + for epoch in range(10): + epoch_start = time.time() + for batch in dataloader: + continue + print(f"Epoch {epoch} / 10 took {time.time() - epoch_start} s.") + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/training/profile_reconstruction.py b/scripts/training/profile_reconstruction.py index 91500d9..3b17b40 100644 --- a/scripts/training/profile_reconstruction.py +++ b/scripts/training/profile_reconstruction.py @@ -23,7 +23,6 @@ def main(): - ### Settings ### parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") parser.add_argument( diff --git a/src/tokamak_foundation_model/data/config/config.yaml b/src/tokamak_foundation_model/data/config/config.yaml index b8266b3..9585910 100644 --- a/src/tokamak_foundation_model/data/config/config.yaml +++ b/src/tokamak_foundation_model/data/config/config.yaml @@ -1,6 +1,6 @@ defaults: - modalities: modalities - - shot_list: train_small + - shot_list: train_additional # These can be overridden from CLI, e.g.: # python generate_data.py shot_list=train diff --git a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml index ede62a5..b9d7f4e 100644 --- a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml +++ b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml @@ -1,138 +1,1248 @@ # Modality definitions for data processing # Each modality specifies how to read from the input HDF5 and write to output -input_data_path: /scratch/gpfs/EKOLEMEN/d3d_fusion_data +input_data_path: /scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_time_series_data output_data_path: /scratch/gpfs/EKOLEMEN/foundation_model -# TODO: merge video data into input_data_path, then remove this -video_data_path: /scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_image_data - -num_workers: 64 +num_workers: 1 signals: - bes: - input_group: bes - input_xkey: axis1 - input_ykey: block0_values - source: default # reads from {shot}.h5 + filterscopes: + tree: D3D + input_key: + - \SPECTROSCOPY::FS01 + - \SPECTROSCOPY::FS02 + - \SPECTROSCOPY::FS03 + - \SPECTROSCOPY::FS04 + - \SPECTROSCOPY::FS05 + - \SPECTROSCOPY::FS06 + - \SPECTROSCOPY::FS07 + - \SPECTROSCOPY::FS08 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT01 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT02 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT03 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT04 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT04 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT05 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT06 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT07 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT08 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT09 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT10 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT11 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT12 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT13 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT14 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT15 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT16 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT17 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT18 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT19 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT20 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT21 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT22 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT23 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT24 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT25 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT26 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT27 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT28 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT29 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT30 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT31 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT32 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT33 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT34 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT35 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT36 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT37 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT38 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT39 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT40 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT41 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT42 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT43 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT44 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT45 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT46 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT47 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT48 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT49 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT50 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT51 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT52 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT53 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT54 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT55 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT56 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT57 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT58 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT59 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT60 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT61 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT62 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT63 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT64 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT65 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT66 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT67 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT68 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT69 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT70 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT71 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT72 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT73 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT74 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT75 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT76 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT77 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT78 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT79 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT80 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT81 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT82 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT83 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT84 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT85 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT86 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT87 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT88 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT89 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT90 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT91 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT92 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT93 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT94 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT95 + - \D3D::TOP.SPECTROSCOPY.FILTERSCOPE.PMT96 + input_xkey: dim0 + input_ykey: data + source: default stft: true - sampling_rate: 500000 - num_channels: 64 + sampling_rate: 10000 + num_channels: 104 - dalpha: - input_group: d_alpha - input_xkey: axis1 - input_ykey: block0_values + cer_ti: + tree: D3D + input_key: + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL01:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL02:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL03:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL04:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL05:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL06:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL07:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL08:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL09:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL10:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL11:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL12:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL13:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL14:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL15:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL16:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL17:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL18:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL19:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL20:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL21:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL22:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL23:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL24:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL25:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL26:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL27:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL28:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL29:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL30:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL31:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL32:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL33:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL34:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL35:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL36:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL37:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL38:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL39:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL40:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL41:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL42:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL43:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL44:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL45:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL46:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL47:TEMP + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL48:TEMP + input_xkey: dim0 + input_ykey: data source: default - stft: true + stft: false + sampling_rate: 100 + num_channels: 48 + + cer_rot: + tree: D3D + input_key: + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL01:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL02:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL03:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL04:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL05:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL06:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL07:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL08:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL09:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL10:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL11:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL12:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL13:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL14:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL15:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL16:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL17:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL18:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL19:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL20:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL21:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL22:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL23:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL24:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL25:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL26:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL27:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL28:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL29:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL30:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL31:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL32:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL33:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL34:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL35:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL36:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL37:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL38:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL39:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL40:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL41:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL42:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL43:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL44:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL45:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL46:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL47:ROT + - \D3D::TOP.IONS.CER.CERAUTO.TANGENTIAL.CHANNEL48:ROT + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 100 + num_channels: 48 + + sxr: + tree: D3D + input_key: + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1F:SX165R1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX165R1S:SX165R1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1F:SX195R1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX195R1S:SX195R1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1F:SX45R1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX45R1S:SX45R1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1F:SX90RM1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RM1S:SX90RM1S32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1F:SX90RP1F32 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S01 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S02 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S03 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S04 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S05 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S06 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S07 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S08 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S09 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S10 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S11 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S12 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S13 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S14 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S15 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S16 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S17 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S18 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S19 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S20 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S21 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S22 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S23 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S24 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S25 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S26 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S27 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S28 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S29 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S30 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S31 + - \D3D::TOP.SPECTROSCOPY.SXR:SX90RP1S:SX90RP1S32 + input_xkey: dim0 + input_ykey: data + source: default + stft: False sampling_rate: 10000 - num_channels: 16 + num_channels: 320 + + neutron_rate: + tree: D3D + input_key: + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONRATE1 + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONRATE3 + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONRATE4 + - \D3D::TOP.IONS.NEUTRONS.FIP:NEUTRONSRATE + input_xkey: dim0 + input_ykey: data + source: default + stft: False + sampling_rate: 40000 + num_channels: 4 mse: - input_group: mse - input_xkey: axis1 - input_ykey: block0_values + tree: D3D + input_key: + - \D3D::TOP.MSE.ANALYSIS_01:MSEP01 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP02 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP03 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP04 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP05 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP06 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP07 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP08 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP09 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP10 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP11 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP12 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP13 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP14 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP15 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP16 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP17 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP18 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP19 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP20 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP21 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP22 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP23 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP24 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP25 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP26 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP27 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP28 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP29 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP30 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP31 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP32 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP33 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP34 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP35 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP36 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP37 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP38 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP39 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP40 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP41 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP42 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP43 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP44 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP45 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP46 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP47 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP48 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP49 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP50 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP51 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP52 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP53 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP54 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP55 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP56 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP57 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP58 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP59 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP60 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP61 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP62 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP63 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP64 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP65 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP66 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP67 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP68 + - \D3D::TOP.MSE.ANALYSIS_01:MSEP69 + input_xkey: dim0 + input_ykey: data source: default stft: false sampling_rate: 100 - num_channels: 36 + num_channels: 69 ts_core_density: - input_group: ts_core_density - input_xkey: axis1 - input_ykey: block0_values + tree: D3D + input_key: + - \D3D::TOP.ELECTRONS.TS.BLESSED.CORE:DENSITY + input_xkey: dim0 + input_ykey: data source: default stft: false sampling_rate: 100 num_channels: 44 - mhr: - input_group: magnetics_high_resolution - input_xkey: axis1 - input_ykey: block0_values + ts_tangential_density: + tree: D3D + input_key: + - \D3D::TOP.ELECTRONS.TS.BLESSED.TANGENTIAL:DENSITY + input_xkey: dim0 + input_ykey: data source: default - stft: true - sampling_rate: 500000 - num_channels: 8 + stft: false + sampling_rate: 100 + num_channels: 10 + + ts_core_temp: + tree: D3D + input_key: + - \D3D::TOP.ELECTRONS.TS.BLESSED.CORE:TEMP + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 100 + num_channels: 44 + + ts_tangential_temp: + tree: D3D + input_key: + - \D3D::TOP.ELECTRONS.TS.BLESSED.TANGENTIAL:TEMP + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 100 + num_channels: 10 ece: - input_group: ece_cali - input_xkey: axis1 - input_ykey: block0_values + tree: D3D + input_key: + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF01 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF02 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF03 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF04 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF05 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF06 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF07 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF08 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF09 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF10 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF11 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF12 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF13 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF14 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF15 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF16 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF17 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF18 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF19 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF20 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF21 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF22 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF23 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF24 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF25 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF26 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF27 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF28 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF29 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF30 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF31 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF32 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF33 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF34 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF35 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF36 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF37 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF38 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF39 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF40 + input_xkey: dim0 + input_ykey: data source: default stft: true sampling_rate: 500000 num_channels: 48 co2: - input_group: co2_density - input_xkey: axis1 - input_ykey: block0_values + tree: D3D + input_key: + - \D3D::TOP.ELECTRONS.BCI.DPD.R0:DENUF + - \D3D::TOP.ELECTRONS.BCI.DPD.V1:DENUF + - \D3D::TOP.ELECTRONS.BCI.DPD.V2:DENUF + - \D3D::TOP.ELECTRONS.BCI.DPD.V3:DENUF + input_xkey: dim0 + input_ykey: data source: default stft: true sampling_rate: 500000 num_channels: 4 - gas: - input_group: gas - input_xkey: axis1 - input_ykey: block0_values + vib: + tree: D3D + input_key: + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_01 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_02 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_03 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_04 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_05 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_06 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_07 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_08 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_09 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_10 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_11 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_12 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_13 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_14 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_15 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_16 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_17 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_18 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_19 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_20 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_21 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_22 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_23 + - \D3D::TOP.SPECTROSCOPY.VB.ZEFF:ZEFF_24 + input_xkey: dim0 + input_ykey: data + source: default + stft: true + sampling_rate: 50 + num_channels: 24 + + bolo: + tree: D3D + input_key: + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L01_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L02_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L03_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L04_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L05_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L06_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L07_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L08_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L09_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L10_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L11_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L12_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L13_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L14_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L15_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L16_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L17_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L18_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L19_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L20_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L21_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L22_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L23_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_L24_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U01_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U02_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U03_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U04_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U05_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U06_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U07_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U08_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U09_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U10_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U11_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U12_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U13_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U14_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U15_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U16_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U17_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U18_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U19_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U20_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U21_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U22_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U23_V + - \D3D::TOP.SPECTROSCOPY.PRAD.BOLOM.RAW:BOL_U24_V + input_xkey: dim0 + input_ykey: data source: default stft: false sampling_rate: 10000 - num_channels: 5 + num_channels: 48 - ech: - input_group: ech - input_xkey: axis1 - input_ykey: block0_values + pinj: + tree: D3D + input_key: + - \D3D::TOP.NB.NB15L:PINJ_15L + - \D3D::TOP.NB.NB15R:PINJ_15R + - \D3D::TOP.NB.NB21L:PINJ_21L + - \D3D::TOP.NB.NB21R:PINJ_21R + - \D3D::TOP.NB.NB30L:PINJ_30L + - \D3D::TOP.NB.NB30R:PINJ_30R + - \D3D::TOP.NB.NB33L:PINJ_33L + - \D3D::TOP.NB.NB33R:PINJ_33R + input_xkey: dim0 + input_ykey: data source: default stft: false sampling_rate: 10000 - num_channels: 11 + num_channels: 8 - pin: - input_group: p_inj - input_xkey: axis1 - input_ykey: block0_values + tinj: + tree: D3D + input_key: + - \D3D::TOP.NB.NB15L:TINJ_15L + - \D3D::TOP.NB.NB15R:TINJ_15R + - \D3D::TOP.NB.NB21L:TINJ_21L + - \D3D::TOP.NB.NB21R:TINJ_21R + - \D3D::TOP.NB.NB30L:TINJ_30L + - \D3D::TOP.NB.NB30R:TINJ_30R + - \D3D::TOP.NB.NB33L:TINJ_33L + - \D3D::TOP.NB.NB33R:TINJ_33R + input_xkey: dim0 + input_ykey: data source: default stft: false sampling_rate: 10000 num_channels: 8 - tin: - input_group: t_inj - input_xkey: axis1 - input_ykey: block0_values + ech: + tree: D3D + input_key: + - \D3D::TOP.RF.ECH.BORIS:ECBORFPWRC + - \D3D::TOP.RF.ECH.CHEWBACCA:ECCHEFPWRC + - \D3D::TOP.RF.ECH.DOROTHY:ECDORFPWRC + - \D3D::TOP.RF.ECH.HAN:ECHANDLPWRC + - \D3D::TOP.RF.ECH.KATYA:ECKATFPWRC + - \D3D::TOP.RF.ECH.LEIA:ECLEIFPWRC + - \D3D::TOP.RF.ECH.LION:ECLIOFPWRC + - \D3D::TOP.RF.ECH.LUKE:ECLUKFPWRC + - \D3D::TOP.RF.ECH.NASA:ECNASFPWRC + - \D3D::TOP.RF.ECH.NATASHA:ECNATFPWRC + - \D3D::TOP.RF.ECH.R2D2:ECR2DFPWRC + - \D3D::TOP.RF.ECH.SCARECROW:ECSCAFPWRC + input_xkey: dim0 + input_ykey: data source: default stft: false sampling_rate: 10000 - num_channels: 8 + num_channels: 12 - bolo: - input_group: bolo - input_xkey: time + gas_flow: + tree: D3D + input_key: + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:FLOW + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:FLOW + input_xkey: dim0 input_ykey: data - source: video # reads from video_data_path/{shot}_image.h5 + source: default stft: false - sampling_rate: 50 - num_channels: 48 - # swap_axes: [0, 2] # swapaxes on ydata + sampling_rate: 10000 + num_channels: 11 + + gas_raw: + tree: D3D + input_key: + - \D3D::TOP.NEUTRALS.GASFLOW.GASA:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASB:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASC:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASD:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.GASE:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB1:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.LOB2:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX1:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX2:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.PFX3:RAW + - \D3D::TOP.NEUTRALS.GASFLOW.UOB:RAW + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 10000 + num_channels: 11 + + ich: + tree: D3D + input_key: + - \D3D::TOP.RF.ICH:ICHPWR + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 10000 + num_channels: 1 irtv: - input_group: irtv - input_xkey: time + tree: IRTV + input_key: + - \IRTV::TOP.IRTV:BIAS_105RM1:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:LOCEN_315RM1:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:LODIV_165RP2:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:LODIV_60RP2:DIGITAL_CAM:DIGITAL_RAW + # - \IRTV::TOP.IRTV:PERI75R0:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:UPCEN_300RP1:DIGITAL_CAM:DIGITAL_RAW + - \IRTV::TOP.IRTV:UPDIV_225RM2:DIGITAL_CAM:DIGITAL_RAW + input_xkey: dim0 input_ykey: data - source: video + source: default stft: false sampling_rate: 50 - num_channels: 48 + num_channels: 7 tangtv: - input_group: tangtv - input_xkey: time + tree: TANGTV + input_key: + - \TANGTV::TOP.TANGTV:LODIV_240RM1:PAR:INTENSIFIED:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:LODIV_240RM1:PAR:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:LODIV_240RM1:PERP:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_225RP1:PERP:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_0RP1:PERP:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_225RP1:PAR:STANDARD:VIDEO_IMAGES + - \TANGTV::TOP.TANGTV:UPDIV_0RP1:PAR:STANDARD:VIDEO_IMAGES + input_xkey: dim0 input_ykey: data - source: video + source: default stft: false sampling_rate: 50 - num_channels: 48 \ No newline at end of file + num_channels: 7 + + mhr: + tree: PTDATA + input_key: + - B1 + - B2 + - B3 + - B4 + - B5 + - B6 + - B7 + - B8 + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 500000 + num_channels: 8 + + mirnov: + tree: PTDATA + input_key: + - MPI1A322D + - MPI3A322D + - MPI5A322D + - MPI89A322D + - MPI79FA322D + - MPI7FA322D + - MPI67A322D + - MPI6NA322D + - MPI1B322D + - MPI3B322D + - MPI5B322D + - MPI89B322D + - MPI79B322D + - MPI7NB322D + - MPI6FB322D + - MPI66M322D + - MPI66M132D + - MPI66B137D + - MPI66M312D + - MPI66B312D + - MPI66M020D + - MPI66M097D + - MPI66M307D + - MPI1A011D + - MPI1A274D + - MPI1A109D + - MPI1A199D + - MPI1A274D + - MPI1A341D + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 500000 + num_channels: 29 + + langmuir: + tree: PTDATA + input_key: + - TPLANG01 + - TPLANG02 + - TPLANG03 + - TPLANG04 + - TPLANG05 + - TPLANG06 + - TPLANG07 + - TPLANG08 + - TPLANG09 + - TPLANG10 + - TPLANG11 + - TPLANG12 + - TPLANG13 + - TPLANG14 + - TPLANG15 + - TPLANG16 + - TPLANG17 + - TPLANG18 + - TPLANG19 + - TPLANG20 + - TPLANG21 + - TPLANG22 + - TPLANG23 + - TPLANG24 + - TPLANG25 + - TPLANG26 + - TPLANG27 + - TPLANG28 + - TPLANG29 + - TPLANG30 + - TPLANG31 + - TPLANG32 + - TPLANG33 + - TPLANG34 + - TPLANG35 + - TPLANG36 + - TPLANG37 + - TPLANG38 + - TPLANG39 + - TPLANG40 + - TPLANG41 + - TPLANG42 + - TPLANG43 + - TPLANG44 + - TPLANG45 + - TPLANG46 + - TPLANG47 + - TPLANG48 + - TPLANG49 + - TPLANG50 + - TPLANG51 + - TPLANG52 + - TPLANG53 + - TPLANG54 + - TPLANG55 + - TPLANG56 + - TPLANG57 + - TPLANG58 + - TPLANG59 + - TPLANG60 + - TPLANG61 + - TPLANG62 + - TPLANG63 + - TPLANG64 + - TPLANG65 + - TPLANG66 + - TPLANG67 + - TPLANG68 + - TPLANG69 + - TPLANG70 + - TPLANG71 + - TPLANG72 + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 500000 + num_channels: 72 + + i_coil: + tree: PTDATA + input_key: + - C19F + - C79F + - C139F + - C199F + - C259F + - C319F + - IU30F + - IU90F + - IU150F + - IU210F + - IU270F + - IU330F + - IL30F + - IL90F + - IL150F + - IL210F + - IL270F + - IL330 + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 50000 + num_channels: 18 + + bes: + tree: PTDATA + input_key: + - BESFU01 + - BESFU02 + - BESFU03 + - BESFU04 + - BESFU05 + - BESFU06 + - BESFU07 + - BESFU08 + - BESFU09 + - BESFU10 + - BESFU11 + - BESFU12 + - BESFU13 + - BESFU14 + - BESFU15 + - BESFU16 + - BESFU17 + - BESFU18 + - BESFU19 + - BESFU20 + - BESFU21 + - BESFU22 + - BESFU23 + - BESFU24 + - BESFU25 + - BESFU26 + - BESFU27 + - BESFU28 + - BESFU29 + - BESFU30 + - BESFU31 + - BESFU32 + - BESFU33 + - BESFU34 + - BESFU35 + - BESFU36 + - BESFU37 + - BESFU38 + - BESFU39 + - BESFU40 + - BESFU41 + - BESFU42 + - BESFU43 + - BESFU44 + - BESFU45 + - BESFU46 + - BESFU47 + - BESFU48 + - BESFU49 + - BESFU50 + - BESFU51 + - BESFU52 + - BESFU53 + - BESFU54 + - BESFU55 + - BESFU56 + - BESFU57 + - BESFU58 + - BESFU59 + - BESFU60 + - BESFU61 + - BESFU62 + - BESFU63 + - BESFU64 + input_xkey: dim0 + input_ykey: data + source: default + stft: false + sampling_rate: 500000 + num_channels: 64 diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index e1ab704..bde9b7f 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -3,22 +3,83 @@ import numpy as np import h5py from pathlib import Path -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional import torch.nn.functional as F import copy -# TODO: implement this for calculation class WelfordTensor: """ - Welford algorithm for computing running statistics on batched multi-channel tensors. - - Computes per-channel statistics by aggregating across batch and all other dimensions. - - For signals (B, C, F, T) or (B, C, 1, T): computes stats per channel → shape (C,) - For profiles (B, S, T): computes stats per spatial point → shape (S,) - For videos (B, T, H, W): computes global stats → shape (1,) + Online Welford algorithm for per-channel statistics on batched tensors. + + Accumulates running mean, variance, minimum, and maximum over an arbitrary + number of :meth:`update` calls without storing the full dataset in memory. + Statistics are computed along the channel axis (axis 1 for 3-D and 4-D + tensors) by aggregating across the batch dimension and all remaining + non-channel dimensions. Batches that contain any ``NaN`` value are + silently skipped. + + The shape of the statistics vectors depends on the input rank: + + ========= =================================== =========== + ``ndim`` Interpretation Stats shape + ========= =================================== =========== + 4 ``(B, C, F, T)`` — spectrograms / ``(C,)`` + time series + 3 ``(B, S, T)`` — profiles ``(S,)`` + ≤ 2 ``(B, T)`` or scalar — video / ``(1,)`` + fallback + ========= =================================== =========== + + Attributes + ---------- + mean : torch.Tensor or None + Running per-channel mean, shape ``(C,)``. ``None`` before the first + :meth:`update` call. + std : torch.Tensor or None + Per-channel sample standard deviation, shape ``(C,)``. Populated + only after :meth:`compute` is called. + min_val : torch.Tensor or None + Running per-channel minimum, shape ``(C,)``. ``None`` before the + first :meth:`update` call. + max_val : torch.Tensor or None + Running per-channel maximum, shape ``(C,)``. ``None`` before the + first :meth:`update` call. + n : int + Total number of scalar samples seen so far (summed over all + non-channel dimensions across all batches). + M2 : torch.Tensor or None + Running sum of squared deviations from the mean (Welford + accumulator), shape ``(C,)``. ``None`` before the first + :meth:`update` call. + initialized : bool + ``True`` once the internal buffers have been allocated on the first + :meth:`update` call. + + Notes + ----- + The parallel (batch) variant of Welford's algorithm is used to combine + each incoming batch with the accumulated state in a single pass + [1]_. All accumulation is done in ``float64`` regardless of the input + dtype to minimise floating-point cancellation errors. + + References + ---------- + .. [1] Welford, B. P. (1962). Note on a method for calculating corrected + sums of squares and products. *Technometrics*, 4(3), 419–420. + https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm + + Examples + -------- + >>> import torch + >>> tracker = WelfordTensor() + >>> for _ in range(10): + ... batch = torch.randn(32, 8, 512, 200) # (B, C, F, T) + ... tracker.update(batch) + >>> stats = tracker.compute() + >>> stats['mean'].shape + (8,) """ def __init__(self): @@ -31,7 +92,26 @@ def __init__(self): self.initialized = False def _initialize(self, value: torch.Tensor): - """Initialize arrays based on first tensor's shape.""" + """ + Allocate accumulator buffers sized to match *value*. + + Called automatically by :meth:`update` on the first non-NaN batch. + Derives the number of channels from the input rank: + + * ``ndim == 4``: channel axis is 1 (spectrograms / time series). + * ``ndim == 3``: channel axis is 1 (profiles / spatial signals). + * ``ndim <= 2``: treated as single-channel (``n_channels = 1``). + + Parameters + ---------- + value : torch.Tensor + First batch tensor, used only to infer ``n_channels``. + Shape must be ``(B, C, ...)`` for 3-D or 4-D inputs. + + Returns + ------- + None + """ # Determine number of channels based on tensor shape (excluding batch dim) if value.ndim == 4: # (batch, channels, freq_bins, time) or (batch, channels, 1, time) @@ -49,22 +129,35 @@ def _initialize(self, value: torch.Tensor): self.mean = torch.zeros(n_channels, dtype=torch.float64) self.M2 = torch.zeros(n_channels, dtype=torch.float64) - self.min_val = torch.full((n_channels,), float('inf'), dtype=torch.float64) - self.max_val = torch.full((n_channels,), float('-inf'), dtype=torch.float64) + self.min_val = torch.full( + (n_channels,), float('inf'), dtype=torch.float64) + self.max_val = torch.full( + (n_channels,), float('-inf'), dtype=torch.float64) self.initialized = True def update(self, value: torch.Tensor): """ - Update statistics with new batched tensor. + Incorporate a new batch into the running statistics. + + Batches that contain any ``NaN`` element are silently skipped. On + the first valid call the accumulator buffers are allocated via + :meth:`_initialize`. Subsequent calls merge the incoming batch + statistics with the accumulated state using the parallel Welford + update rule. Parameters ---------- value : torch.Tensor - Input tensor of shape: - - (batch, channels, freq_bins, time) for spectrograms - - (batch, channels, 1, time) for time series - - (batch, spatial_points, time) for profiles - - (batch, time, height, width) for videos + Batched input tensor. Supported shapes: + + * ``(B, C, F, T)`` — spectrograms or multi-channel time series. + * ``(B, C, 1, T)`` — single-frequency time series. + * ``(B, S, T)`` — spatial profiles. + * ``(B, T, H, W)`` — video frames (global statistics). + + Returns + ------- + None """ # Skip if contains NaN if torch.isnan(value).any(): @@ -81,9 +174,8 @@ def update(self, value: torch.Tensor): if value.ndim == 4 and value.shape[1] == self.mean.shape[0]: # (batch, channels, freq_bins, time) → flatten batch, freq, time # (B, C, F, T) → (C, B*F*T) - batch_size = value.shape[0] n_channels = value.shape[1] - value_flat = value.permute(1, 0, 2, 3).reshape(n_channels, -1) # (C, B*F*T) + value_flat = value.permute(1, 0, 2, 3).reshape(n_channels, -1) # Per-channel mean, min, max batch_mean = value_flat.mean(dim=1) @@ -99,7 +191,7 @@ def update(self, value: torch.Tensor): # (batch, spatial_points, time) → flatten batch, time # (B, S, T) → (S, B*T) n_channels = value.shape[1] - value_flat = value.permute(1, 0, 2).reshape(n_channels, -1) # (S, B*T) + value_flat = value.permute(1, 0, 2).reshape(n_channels, -1) batch_mean = value_flat.mean(dim=1) batch_min = value_flat.min(dim=1).values @@ -142,7 +234,17 @@ def update(self, value: torch.Tensor): self.max_val = torch.maximum(self.max_val, batch_max) def _compute_std(self): - """Compute standard deviation from M2.""" + """ + Derive sample standard deviation from the Welford M2 accumulator. + + Uses Bessel's correction (``n - 1``) when more than one sample has + been seen; falls back to zeros when ``n <= 1`` to avoid division by + zero. The result is written to :attr:`std` in-place. + + Returns + ------- + None + """ if self.n > 1: self.std = torch.sqrt(self.M2 / (self.n - 1)) else: @@ -150,16 +252,25 @@ def _compute_std(self): def compute(self): """ - Compute final statistics. + Finalise and return all accumulated statistics as NumPy arrays. + + Calls :meth:`_compute_std` internally to derive the standard + deviation from the Welford M2 accumulator before returning. Returns ------- dict - Dictionary with numpy arrays: - - 'mean': per-channel mean - - 'std': per-channel standard deviation - - 'min_val': per-channel minimum - - 'max_val': per-channel maximum + Dictionary with the following keys, each mapping to a + ``numpy.ndarray`` of shape ``(C,)``: + + ``'mean'`` + Per-channel arithmetic mean. + ``'std'`` + Per-channel sample standard deviation (Bessel-corrected). + ``'min_val'`` + Per-channel minimum value seen across all batches. + ``'max_val'`` + Per-channel maximum value seen across all batches. """ self._compute_std() @@ -187,7 +298,7 @@ def compute_preprocessing_stats( from tqdm import tqdm combined = ConcatDataset(datasets) - dataloader = DataLoader(combined, batch_size=32, collate_fn=collate_fn, num_workers=1) + dataloader = DataLoader(combined, batch_size=32, collate_fn=collate_fn, num_workers=32) # Get signal names from first dataset signal_configs = datasets[0].SIGNAL_CONFIGS diff --git a/src/tokamak_foundation_model/data/prepare_data.py b/src/tokamak_foundation_model/data/prepare_data.py deleted file mode 100644 index a53b95d..0000000 --- a/src/tokamak_foundation_model/data/prepare_data.py +++ /dev/null @@ -1,247 +0,0 @@ -import numpy as np -import h5py -import hydra -import logging -from multiprocessing import Pool -from functools import partial -from omegaconf import DictConfig, OmegaConf -from pathlib import Path -from tqdm.auto import tqdm -from scipy.interpolate import interp1d -import os - - -log = logging.getLogger(__name__) - -# ── hardcoded until video data is merged into the main data path ── -_VIDEO_DATA_PATH = Path("/scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_image_data") - - -def _resample_time_series(data, time, target_frequency): - """ - Resample non-uniformly sampled time series to uniform sampling. - - Parameters: - ----------- - data : np.ndarray, shape (n_samples, ...) - Time series data - time : np.ndarray, shape (n_samples,) - Time axis (can be non-uniform) - target_frequency : float - Desired sampling frequency in Hz - - Returns: - -------- - resampled_data : np.ndarray - Uniformly resampled data - new_time : np.ndarray - New uniform time axis - """ - if len(data) <= 1: - return time.copy(), data.copy() - - # Calculate target sampling period - dt = 1.0 / target_frequency - - # Create uniform time grid - n_samples = int(np.ceil((time[-1] - time[0]) / dt)) + 1 - new_time = time[0] + np.arange(n_samples) * dt - - # Handle multi-dimensional data - original_shape = data.shape - if data.ndim > 1: - # Flatten all dimensions except the first (time) - data_flat = data.reshape(data.shape[0], -1) - resampled_flat = np.full((len(new_time), data_flat.shape[1]), np.nan) - - # Interpolate each channel, handling NaNs - for i in range(data_flat.shape[1]): - # Find valid (non-NaN) data points - valid_mask = ~np.isnan(data_flat[:, i]) - - if np.sum(valid_mask) >= 2: # Need at least 2 points to interpolate - valid_time = time[valid_mask] - valid_data = data_flat[valid_mask, i] - - # Only interpolate within the range of valid data - interpolator = interp1d(valid_time, valid_data, kind='linear', - bounds_error=False, fill_value=np.nan) - resampled_flat[:, i] = interpolator(new_time) - # else: remains NaN (initialized above) - - # Reshape back to original dimensions (except time axis) - new_shape = (len(new_time),) + original_shape[1:] - resampled_data = resampled_flat.reshape(new_shape) - else: - # 1D case - valid_mask = ~np.isnan(data) - - if np.sum(valid_mask) >= 2: - valid_time = time[valid_mask] - valid_data = data[valid_mask] - - interpolator = interp1d(valid_time, valid_data, kind='linear', - bounds_error=False, fill_value=np.nan) - resampled_data = interpolator(new_time) - else: - # Not enough valid data to interpolate - resampled_data = np.full(len(new_time), np.nan) - - return new_time, resampled_data - - -def _get_valid_shots( - shot_list: list[int], - input_data_path: Path, - video_data_path: Path, -) -> list[int]: - """Return only shots that have files in *both* the main data path and the - video data path. Expects ``{shot}.h5`` in input_data_path and - ``{shot}_image.h5`` in video_data_path.""" - - main_shots = { - int(p.stem) - for p in input_data_path.glob("*.h5") - if p.stem.isdigit() - } - video_shots = { - int(p.stem.replace("_image", "")) - for p in video_data_path.glob("*_image.h5") - } - available = main_shots & video_shots - requested = set(shot_list) - valid = sorted(requested & available) - - n_missing = len(requested) - len(valid) - if n_missing: - log.warning( - f"{n_missing}/{len(requested)} requested shots missing from one " - f"or both data paths – skipped" - ) - log.info(f"{len(valid)} shots available in both paths") - return valid - - -def _process_shot(shot: int, cfg_dict: dict) -> str | None: - """Worker function executed in a child process. - - Args: - shot: Shot number. - cfg_dict: Plain dict (not DictConfig – must be picklable). - - Returns: - None on success, or an error message string on failure. - """ - try: - input_data_path = Path(cfg_dict["input_data_path"]) - video_data_path = Path( - cfg_dict.get("video_data_path", str(_VIDEO_DATA_PATH))) - output_data_path = Path(cfg_dict["output_data_path"]) - output_data_path.mkdir(parents=True, exist_ok=True) - - output_file = output_data_path / f"{shot}_processed.h5" - - signals = cfg_dict["signals"] - - # ── group signals by source ── - source_to_signals: dict[str, list[tuple[str, dict]]] = {} - for abbr, sig_cfg in signals.items(): - source = sig_cfg.get("source", "default") - source_to_signals.setdefault(source, []).append((abbr, sig_cfg)) - - # Map source key → input filename - source_file_map = { - "default": input_data_path / f"{shot}.h5", - "video": video_data_path / f"{shot}_image.h5", - } - - # ── read all signals ── - read_data: dict[str, tuple[np.ndarray, np.ndarray]] = {} - - for source_key, sigs in source_to_signals.items(): - fpath = source_file_map.get(source_key) - if fpath is None or not fpath.exists(): - continue - - with h5py.File(fpath, "r") as f: - for abbr, sig_cfg in sigs: - grp_name = sig_cfg["input_group"] - if grp_name not in f: - continue - - xdata = f[grp_name][sig_cfg["input_xkey"]][:] - ydata = f[grp_name][sig_cfg["input_ykey"]][:] - - if sig_cfg.get("swap_axes") is not None: - ydata = ydata.swapaxes(*sig_cfg["swap_axes"]) - - xdata, ydata = _resample_time_series( - data=ydata, - time=xdata / 1000, - target_frequency=sig_cfg["sampling_rate"]) - - read_data[abbr] = (xdata * 1000, ydata) - - if not read_data: - return f"shot {shot}: no data read – skipped" - - # ── write processed file ── - with h5py.File(output_file, "w") as f: - for abbr, (xdata, ydata) in read_data.items(): - grp = f.create_group(abbr) - grp.create_dataset("xdata", data=xdata, dtype='f8') - grp.create_dataset("ydata", data=ydata, dtype='f8') - - os.chmod(output_file, 0o664) - return None # success - - except Exception as e: - log.info(f"shot {shot}: {type(e).__name__}: {e}") - return f"shot {shot}: {type(e).__name__}: {e}" - - -@hydra.main(version_base=None, config_path="config", config_name="config") -def main(cfg: DictConfig) -> None: - log.info(f"Config:\n{OmegaConf.to_yaml(cfg)}") - - mod_cfg = cfg.modalities - input_data_path = Path(mod_cfg.input_data_path) - video_data_path = Path( - mod_cfg.get("video_data_path", str(_VIDEO_DATA_PATH))) - num_workers = mod_cfg.get("num_workers", 8) - - # ── filter to shots that exist in both paths ── - shots = _get_valid_shots( - shot_list=list(cfg.shot_list.shots), - input_data_path=input_data_path, - video_data_path=video_data_path, - ) - - if not shots: - log.error("No valid shots found – exiting.") - return - - # Convert to plain dict so it's picklable for multiprocessing - cfg_dict = OmegaConf.to_container(mod_cfg, resolve=True) - - log.info(f"Processing {len(shots)} shots with {num_workers} workers") - - worker = partial(_process_shot, cfg_dict=cfg_dict) - - errors = [] - - with Pool(processes=num_workers) as pool: - for i, err in enumerate( - tqdm(pool.imap_unordered(worker, shots), total=len(shots))): - if err is not None: - log.error(err) - errors.append(err) - - log.info( - f"Done. {len(shots) - len(errors)}/{len(shots)} succeeded, " - f"{len(errors)} failed." - ) - - -if __name__ == "__main__": - main() From aa039664269a1c744a1706c1bb91f817034df048 Mon Sep 17 00:00:00 2001 From: renierts Date: Wed, 25 Feb 2026 16:01:37 -0500 Subject: [PATCH 23/30] Generalized make_preprocessing_stats.py and made the function compute_preprocessing_stats more transparent. Bugfix in modalities.yaml - Channels were missing in ECE. --- .../data_preparation/make_processing_stats.py | 19 +- .../data/config/modalities/modalities.yaml | 8 + .../data/data_loader.py | 784 +++++++++++++++--- 3 files changed, 708 insertions(+), 103 deletions(-) diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index 55f329b..6958b8d 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -5,7 +5,7 @@ def main(): hdf5_files = sorted( Path("/scratch/gpfs/EKOLEMEN/foundation_model/" - ).glob("[0-9]*_processed.h5") + ).glob("20000[0-7]_processed.h5") ) # hdf5_files = sorted( @@ -13,10 +13,17 @@ def main(): # ) all_input_signals = [ - "mhr", "ece", "co2", "bes", # spectrograms - "gas", "ech", "pin", "tin", # actuators - "d_alpha", "mse", "ts_core_density", # diagnostics - "bolo", "irtv", "tangtv", # videos + # STFT spectrograms + "mhr", "ece", "co2", + # actuators / gas / heating + "gas", "ech", "pin", "tin", "gas_flow", "gas_raw", "ich", + # diagnostics + "filterscopes", "vib", "mse", "ts_core_density", "ts_core_temp", + "ts_tangential_density", "ts_tangential_temp", "cer_ti", "cer_rot", + "sxr", "neutron_rate", "bolo_raw", "mirnov", "langmuir", "i_coil", + "bes", + # cameras + "irtv", "tangtv", # "text", # metadata ] @@ -27,7 +34,7 @@ def main(): target_signals=all_input_signals, ) for f in hdf5_files] - stats = compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') + compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') if __name__ == "__main__": diff --git a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml index b9d7f4e..9b6e0f2 100644 --- a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml +++ b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml @@ -748,6 +748,14 @@ signals: - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF38 - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF39 - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF40 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF41 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF42 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF43 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF44 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF45 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF46 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF47 + - \D3D::TOP.ELECTRONS.ECE.TECEF:TECEF48 input_xkey: dim0 input_ykey: data source: default diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index bde9b7f..6a0359b 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -170,7 +170,8 @@ def update(self, value: torch.Tensor): # Convert to float64 for numerical stability value = value.to(dtype=torch.float64) - # Compute per-channel statistics by flattening batch and all non-channel dims + # Compute per-channel statistics by flattening batch + # and all non-channel dims if value.ndim == 4 and value.shape[1] == self.mean.shape[0]: # (batch, channels, freq_bins, time) → flatten batch, freq, time # (B, C, F, T) → (C, B*F*T) @@ -256,11 +257,13 @@ def compute(self): Calls :meth:`_compute_std` internally to derive the standard deviation from the Welford M2 accumulator before returning. + Returns ``None`` if :meth:`update` was never called. Returns ------- - dict - Dictionary with the following keys, each mapping to a + dict or None + ``None`` if no data was ever seen. Otherwise a dictionary + with the following keys, each mapping to a ``numpy.ndarray`` of shape ``(C,)``: ``'mean'`` @@ -272,6 +275,9 @@ def compute(self): ``'max_val'`` Per-channel maximum value seen across all batches. """ + if not self.initialized: + return None + self._compute_std() return { @@ -283,38 +289,86 @@ def compute(self): def compute_preprocessing_stats( - datasets, - output_path="preprocessing_stats.pt", - num_samples=1000 -): - """Compute preprocessing statistics across multiple datasets. - - Args: - datasets: List of TokamakH5Dataset instances - output_path: Where to save statistics - num_samples: Number of samples per dataset to use + datasets: "list[TokamakH5Dataset]", + output_path: str | Path = "preprocessing_stats.pt", + num_samples: int = 1000, + batch_size: int = 32, + num_workers: int = 1, +) -> dict[str, dict[str, np.ndarray]]: + """ + Compute per-modality preprocessing statistics over a collection of + datasets. + + For each dataset, draws a random subset of up to *num_samples* chunks, + concatenates the subsets, then accumulates running statistics with + :class:`WelfordTensor`. The result is saved to *output_path* via + :func:`torch.save`. Only modalities that actually appear in the loaded + batches are included in the output. + + Parameters + ---------- + datasets : list of TokamakH5Dataset + One or more dataset instances whose data will be concatenated. + Signal and movie configurations are read from ``datasets[0]``. + output_path : str or Path, optional + Filesystem path for the saved ``.pt`` statistics file. + Default is ``"preprocessing_stats.pt"``. + num_samples : int, optional + Maximum number of chunks to draw randomly from *each* dataset. + Default is ``1000``. + batch_size : int, optional + Batch size for the internal DataLoader. Default is ``32``. + num_workers : int, optional + Number of DataLoader worker processes. Default is ``1``. + + Returns + ------- + dict[str, dict[str, numpy.ndarray]] + Nested dictionary ``{modality_name: stats}``, where *stats* is the + dictionary returned by :meth:`WelfordTensor.compute`: + + ``'mean'`` + Per-channel arithmetic mean, shape ``(C,)``. + ``'std'`` + Per-channel sample standard deviation, shape ``(C,)``. + ``'min_val'`` + Per-channel minimum, shape ``(C,)``. + ``'max_val'`` + Per-channel maximum, shape ``(C,)``. """ - from torch.utils.data import ConcatDataset + from torch.utils.data import ConcatDataset, Subset from tqdm import tqdm - combined = ConcatDataset(datasets) - dataloader = DataLoader(combined, batch_size=32, collate_fn=collate_fn, num_workers=32) + # Draw a random subset from each dataset to stay within num_samples + sampled = [] + for ds in datasets: + n = min(num_samples, len(ds)) + indices = torch.randperm(len(ds))[:n].tolist() + sampled.append(Subset(ds, indices)) + + combined = ConcatDataset(sampled) + dataloader = DataLoader( + combined, batch_size=batch_size, collate_fn=collate_fn, + num_workers=num_workers) - # Get signal names from first dataset - signal_configs = datasets[0].SIGNAL_CONFIGS - movie_configs = datasets[0].MOVIE_CONFIGS + # Use instance-level configs (deep copies that may have been modified) + signal_configs = datasets[0].signal_configs + movie_configs = datasets[0].movie_configs - welford_stats = {cfg.name: WelfordTensor() for cfg in signal_configs + movie_configs} + welford_stats = { + cfg.name: WelfordTensor() for cfg in signal_configs + movie_configs} for batch in tqdm(dataloader): for modality_name, tensor in batch.items(): - # Update statistics + if modality_name not in welford_stats: + continue welford_stats[modality_name].update(tensor) - # Compute final statistics + # Only include trackers that received data final_stats = { modality: tracker.compute() for modality, tracker in welford_stats.items() + if tracker.initialized } torch.save(final_stats, output_path) @@ -324,9 +378,49 @@ def compute_preprocessing_stats( @dataclass class PreprocessConfig: - """Preprocessing configuration.""" + """ + Configuration for a signal preprocessing transformation. - method: str = "none" # "none", "standardize", "normalize", "log_standardize" + Specifies which normalisation strategy to apply to a tensor before it is + fed into the model. Statistics (*mean*, *std*, *min_val*, *max_val*) + are populated at runtime from pre-computed dataset statistics (see + :func:`compute_preprocessing_stats`). + + Parameters + ---------- + method : str, optional + Transformation to apply. One of: + + ``'none'`` + Pass the tensor through unchanged. + ``'standardize'`` + Zero-mean, unit-variance scaling: + ``(x - mean) / (std + eps)``. + ``'normalize'`` + Min-max scaling to ``[0, 1]``: + ``(x - min_val) / (max_val - min_val + eps)``. + ``'log_standardize'`` + Apply ``log10(x + 1)``, then standardize. + ``'log'`` + Apply ``log10(x + 1)`` only. + + Default is ``'none'``. + mean : float or None, optional + Per-channel mean used by ``'standardize'`` and + ``'log_standardize'``. Default is ``None``. + std : float or None, optional + Per-channel standard deviation used by ``'standardize'`` and + ``'log_standardize'``. Default is ``None``. + min_val : float or None, optional + Per-channel minimum used by ``'normalize'``. Default is ``None``. + max_val : float or None, optional + Per-channel maximum used by ``'normalize'``. Default is ``None``. + eps : float, optional + Small constant added to denominators for numerical stability. + Default is ``1e-8``. + """ + + method: str = "none" mean: Optional[float] = None std: Optional[float] = None min_val: Optional[float] = None @@ -336,14 +430,44 @@ class PreprocessConfig: @dataclass class SignalConfig: - """Configuration for a single signal/diagnostic.""" + """ + Configuration for a single time-series or spectrogram diagnostic. + + Collects all parameters needed to load, resample, and preprocess one + modality from an HDF5 file produced by the data-preparation pipeline. + + Parameters + ---------- + name : str + Unique identifier for this modality; used as the dictionary key + in the batch returned by :class:`TokamakH5Dataset`. + hdf5_keys : list of str + Ordered list of HDF5 group paths to search for the signal data. + The first path that exists in the file is used. + num_channels : int + Expected number of signal channels (``C``). + target_fs : float + Target sampling frequency in Hz. The raw signal is resampled to + this rate before being returned. + apply_stft : bool + If ``True``, compute an STFT magnitude spectrogram after loading, + yielding output shape ``(C, F, T)``. If ``False``, the signal is + returned as ``(C, 1, T)``. + channels_to_use : slice + Optional slice to select specific channels + preprocess : PreprocessConfig, optional + Preprocessing transformation applied after the STFT (or + pass-through). Defaults to :class:`PreprocessConfig` with + ``method='none'``. + """ name: str hdf5_keys: list[str] num_channels: int target_fs: float apply_stft: bool - preprocess: PreprocessConfig = None # Add preprocessing config + channels_to_use: slice = field(default_factory=lambda: slice(0, -1)) + preprocess: PreprocessConfig = None def __post_init__(self): if self.preprocess is None: @@ -352,7 +476,35 @@ def __post_init__(self): @dataclass class MovieConfig: - """Configuration for a movie/video diagnostic.""" + """ + Configuration for a video / camera diagnostic. + + Collects all parameters needed to load, resample, and preprocess one + movie modality from an HDF5 file produced by the data-preparation + pipeline. + + Parameters + ---------- + name : str + Unique identifier for this modality; used as the dictionary key + in the batch returned by :class:`TokamakH5Dataset`. + hdf5_keys : list of str + Ordered list of HDF5 group paths to search for the movie data. + The first path that exists in the file is used. + channels : int + Number of colour channels (e.g. ``1`` for grayscale, ``3`` for + RGB). + target_fps : int + Target frame rate in frames per second. The raw video is + resampled to this rate via trilinear interpolation. + height : int + Output frame height in pixels after spatial resampling. + width : int + Output frame width in pixels after spatial resampling. + preprocess : PreprocessConfig, optional + Preprocessing transformation applied to the video tensor. + Defaults to :class:`PreprocessConfig` with ``method='none'``. + """ name: str # Key in output dict hdf5_keys: list[str] # Possible HDF5 paths to search @@ -369,17 +521,130 @@ def __post_init__(self): class TokamakH5Dataset(Dataset): """ - Dataset for loading multi-modal tokamak data from HDF5 files. + PyTorch Dataset for multi-modal tokamak plasma diagnostics stored in HDF5. + + Each item corresponds to a fixed-duration time window (chunk) drawn from a + single shot file. The processing pipeline for every chunk is: + + 1. Load raw signal / movie data at the native sampling rate from HDF5. + 2. Optionally compute an STFT magnitude spectrogram (signals only). + 3. Resample to the modality's target frequency via linear or trilinear + interpolation. + 4. Apply the configured preprocessing transformation + (see :class:`PreprocessConfig`). + + Two operating modes are supported: + + **Standard mode** (``prediction_mode=False``) + Returns a flat dictionary ``{modality_name: tensor}`` covering the + half-open interval ``[t_start, t_start + chunk_duration_s)``. + + **Prediction mode** (``prediction_mode=True``) + Loads an extended window of + ``chunk_duration_s + prediction_horizon_s`` seconds, processes it + jointly, then splits into + ``{"inputs": {…}, "targets": {…}}``. - Processing pipeline: - 1. Load raw data at native sampling rate - 2. Apply processing (STFT or nothing) - 3. Resample to target time frames + Parameters + ---------- + hdf5_path : str + Path to a preprocessed HDF5 shot file (output of the + data-preparation pipeline). + chunk_duration_s : float, optional + Duration of each time window in seconds. Default is ``0.5``. + n_fft : int, optional + FFT size used for STFT computation. Determines the number of + frequency bins: ``n_fft // 2 + 1``. Default is ``1024``. + hop_length : int, optional + STFT hop size in samples. Default is ``256``. + preprocessing_stats : dict or None, optional + Nested statistics dictionary as returned by + :func:`compute_preprocessing_stats`. When provided, the per-modality + statistics are injected into the corresponding + :class:`PreprocessConfig` instances. Default is ``None`` + (no statistics applied). + prediction_mode : bool, optional + If ``True``, operate in prediction mode. Default is ``False``. + prediction_horizon_s : float, optional + Duration of the prediction target window in seconds. Only used + when ``prediction_mode=True``. Default is ``0.2``. + input_signals : list of str or None, optional + Modality names to include in the returned batch (or in the + ``'inputs'`` dict in prediction mode). Defaults to + ``['ece', 'co2', 'mhr']``. + target_signals : list of str or None, optional + Modality names to include in the ``'targets'`` dict in prediction + mode. Defaults to ``['d_alpha', 'mse', 'ts_core_density']``. + + Attributes + ---------- + signal_configs : list of SignalConfig + Per-instance deep copy of :attr:`SIGNAL_CONFIGS`, updated with + any statistics from *preprocessing_stats*. + movie_configs : list of MovieConfig + Per-instance deep copy of :attr:`MOVIE_CONFIGS`. + hdf5_path : Path + Resolved path to the HDF5 file. + duration : float + Total shot duration from t = 0 in seconds, as inferred from the + HDF5 time axes. + t0_indices : dict + Mapping ``{modality_name: {'index': int, 'time_s': float}}`` + giving the HDF5 array index and exact timestamp (seconds) of + t = 0 for each modality. + length : int + Number of non-overlapping chunks available (i.e. ``__len__``). + n_freq_bins : int + Number of STFT frequency bins: ``n_fft // 2 + 1``. + stft_window : torch.Tensor + Hann window tensor of length ``n_fft`` used for STFT computation. - For prediction mode: - - Loads extended window (input_duration + prediction_horizon) - - Processes entire window jointly - - Splits into input and target frames + Notes + ----- + The class-level :attr:`SIGNAL_CONFIGS` and :attr:`MOVIE_CONFIGS` lists + define the full set of supported diagnostics: + + **Signals** (``SIGNAL_CONFIGS``) + + ========================== ======== ========== ===== ================== + Name Channels Target fs STFT Preprocessing + ========================== ======== ========== ===== ================== + ``mhr`` 8 500 kHz yes log + ``ece`` 48 500 kHz yes log + ``co2`` 4 500 kHz yes log + ``gas`` 5 10 kHz no none + ``ech`` 11 10 kHz no none + ``pin`` 8 10 kHz no standardize + ``tin`` 8 10 kHz no none + ``mse`` 69 100 Hz no none + ``ts_core_density`` 44 100 Hz no log + ``filterscopes`` 104 10 kHz yes log + ``cer_ti`` 48 100 Hz no log + ``cer_rot`` 48 100 Hz no none + ``sxr`` 320 10 kHz no log + ``neutron_rate`` 4 40 kHz no log + ``ts_tangential_density`` 10 100 Hz no log + ``ts_core_temp`` 44 100 Hz no log + ``ts_tangential_temp`` 10 100 Hz no log + ``vib`` 24 50 Hz yes log + ``bolo_raw`` 48 10 kHz no log + ``gas_flow`` 11 10 kHz no none + ``gas_raw`` 11 10 kHz no none + ``ich`` 1 10 kHz no none + ``mirnov`` 29 500 kHz no log + ``langmuir`` 72 500 kHz no log + ``i_coil`` 18 50 kHz no none + ``bes`` 64 500 kHz no log + ========================== ======== ========== ===== ================== + + **Movies** (``MOVIE_CONFIGS``) + + =========== === ======= ========= + Name FPS Height Width + =========== === ======= ========= + ``irtv`` 50 513 640 + ``tangtv`` 50 240 720 + =========== === ======= ========= """ # Define all signal configurations with preprocessing @@ -390,7 +655,8 @@ class TokamakH5Dataset(Dataset): 8, 500e3, apply_stft=True, - preprocess=PreprocessConfig(method="log_standardize"), + channels_to_use=slice(2, 8), # Use only the first 8 channels + preprocess=PreprocessConfig(method="log"), ), SignalConfig( "ece", @@ -398,7 +664,8 @@ class TokamakH5Dataset(Dataset): 48, 500e3, apply_stft=True, - preprocess=PreprocessConfig(method="log_standardize"), + channels_to_use=slice(0, 40), # Use only the first 40 channels + preprocess=PreprocessConfig(method="log"), ), SignalConfig( "co2", @@ -408,14 +675,6 @@ class TokamakH5Dataset(Dataset): apply_stft=True, preprocess=PreprocessConfig(method="log"), ), - SignalConfig( - "d_alpha", - ["dalpha"], - 6, - 10e3, - apply_stft=False, - preprocess=PreprocessConfig(method="standardize"), - ), SignalConfig( "gas", ["gas"], @@ -448,7 +707,6 @@ class TokamakH5Dataset(Dataset): apply_stft=False, preprocess=PreprocessConfig(method="none"), ), - # TODO: Include Gas as additional actuator!!! SignalConfig( "mse", ["mse"], @@ -465,17 +723,153 @@ class TokamakH5Dataset(Dataset): apply_stft=False, preprocess=PreprocessConfig(method="log"), ), + # --- groups below added from modalities.yaml --- + SignalConfig( + "filterscopes", + ["filterscopes"], + 104, + 10e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "cer_ti", + ["cer_ti"], + 48, + 1e2, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "cer_rot", + ["cer_rot"], + 48, + 1e2, + apply_stft=False, + preprocess=PreprocessConfig(method="none"), + ), + SignalConfig( + "sxr", + ["sxr"], + 320, + 10e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "neutron_rate", + ["neutron_rate"], + 4, + 40e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "ts_tangential_density", + ["ts_tangential_density"], + 10, + 1e2, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "ts_core_temp", + ["ts_core_temp"], + 44, + 1e2, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "ts_tangential_temp", + ["ts_tangential_temp"], + 10, + 1e2, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "vib", + ["vib"], + 24, + 50, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "bolo_raw", + ["bolo"], + 48, + 10e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "gas_flow", + ["gas_flow"], + 11, + 10e3, + apply_stft=False, + preprocess=PreprocessConfig(method="none"), + ), + SignalConfig( + "gas_raw", + ["gas_raw"], + 11, + 10e3, + apply_stft=False, + preprocess=PreprocessConfig(method="none"), + ), + SignalConfig( + "ich", + ["ich"], + 1, + 10e3, + apply_stft=False, + preprocess=PreprocessConfig(method="none"), + ), + SignalConfig( + "mirnov", + ["mirnov"], + 29, + 500e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "langmuir", + ["langmuir"], + 72, + 500e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), + SignalConfig( + "i_coil", + ["i_coil"], + 18, + 50e3, + apply_stft=False, + preprocess=PreprocessConfig(method="none"), + ), + SignalConfig( + "bes", + ["bes"], + 64, + 500e3, + apply_stft=False, + preprocess=PreprocessConfig(method="log"), + ), ] MOVIE_CONFIGS = [ - MovieConfig("bolo", ["bolo"], 1, 50, 80, 120), MovieConfig("irtv", ["irtv"], 1, 50, 513, 640), MovieConfig("tangtv", ["tangtv"], 1, 50, 240, 720), ] def __init__( self, - hdf5_path: str, + hdf5_path: str | Path, chunk_duration_s: float = 0.5, n_fft: int = 1024, hop_length: int = 256, @@ -489,7 +883,10 @@ def __init__( self.signal_configs = copy.deepcopy(self.SIGNAL_CONFIGS) self.movie_configs = copy.deepcopy(self.MOVIE_CONFIGS) - self.hdf5_path = Path(hdf5_path) + if isinstance(hdf5_path, str): + self.hdf5_path = Path(hdf5_path) + else: + self.hdf5_path = hdf5_path self.chunk_duration_s = chunk_duration_s self.n_fft = n_fft self.hop_length = hop_length @@ -499,7 +896,8 @@ def __init__( self.prediction_mode = prediction_mode self.prediction_horizon_s = prediction_horizon_s self.input_signals = input_signals or ["ece", "co2", "mhr"] - self.target_signals = target_signals or ["d_alpha", "mse", "ts_core_density"] + self.target_signals = ( + target_signals or ["d_alpha", "mse", "ts_core_density"]) if not self.hdf5_path.exists(): raise FileNotFoundError(f"HDF5 file not found: {self.hdf5_path}") @@ -508,7 +906,8 @@ def __init__( self.h5_file = None try: with h5py.File(self.hdf5_path, "r") as f: - self.duration, self.t0_indices = self._compute_duration_and_t0_indices(f) + self.duration, self.t0_indices = \ + self._compute_duration_and_t0_indices(f) except OSError as e: print(self.hdf5_path) raise e @@ -516,9 +915,11 @@ def __init__( if self.prediction_mode: total_window = self.chunk_duration_s + self.prediction_horizon_s max_time = self.duration - total_window - self.length = max(1, int(np.floor(max_time / self.chunk_duration_s))) + self.length = max( + 1, int(np.floor(max_time / self.chunk_duration_s))) else: - self.length = max(1, int(np.ceil(self.duration / self.chunk_duration_s))) + self.length = max( + 1, int(np.ceil(self.duration / self.chunk_duration_s))) self.n_freq_bins = n_fft // 2 + 1 self.stft_window = torch.hann_window(n_fft) @@ -530,14 +931,14 @@ def _find_t0_index(self, xdata_ms: np.ndarray) -> tuple[int, float]: Parameters ---------- xdata_ms : np.ndarray - Array of timestamps in milliseconds + Array of timestamps in milliseconds, assumed sorted ascending. Returns ------- - tuple[int, float] - (index, actual_time_ms) where: - - index: Index closest to t=0, or -1 if all data is before t=0 - - actual_time_ms: The actual timestamp at that index + index : int + Index closest to t=0, or ``-1`` if all data is before t=0. + actual_time_ms : float + The actual timestamp at that index, in milliseconds. """ if len(xdata_ms) == 0: return -1, 0.0 @@ -570,17 +971,33 @@ def _find_t0_index(self, xdata_ms: np.ndarray) -> tuple[int, float]: return idx, xdata_ms[idx] - def _compute_duration_and_t0_indices(self, f: h5py.File) -> tuple[float, dict]: + def _compute_duration_and_t0_indices( + self, + f: h5py.File + ) -> tuple[float, dict]: """ - Compute duration from t=0 and store info about where t=0 occurs for each signal. + Compute shot duration from t=0 and locate the t=0 index per signal. + + Iterates over all signal and movie configurations, reads the + ``xdata`` timestamps from the HDF5 file, finds the first sample at + or after t=0, and accumulates the maximum duration across all + available diagnostics. + + Parameters + ---------- + f : h5py.File + Open HDF5 file handle for the shot. Returns ------- - tuple[float, dict] - (max_duration_from_t0, {signal_name: {'index': int, 'time_s': float}}) - where: - - 'index': first index where xdata >= 0 - - 'time_s': actual time value (in seconds) at that index + max_duration : float + Duration in seconds from t=0 to the last sample, across all + signals and movies. Guaranteed to be at least 1.0 s. + t0_indices : dict[str, dict[str, int | float]] + Mapping from signal/movie name to a dict with keys: + + - ``'index'``: first HDF5 sample index where ``xdata >= 0``. + - ``'time_s'``: actual timestamp at that index, in seconds. """ max_duration = 0.0 t0_indices = {} @@ -656,7 +1073,19 @@ def _compute_duration_and_t0_indices(self, f: h5py.File) -> tuple[float, dict]: return max(max_duration, 1.0), t0_indices def _update_preprocessing_stats(self): - """Update preprocessing configs with loaded statistics.""" + """ + Propagate loaded statistics into each signal's preprocessing config. + + Reads ``self.preprocessing_stats`` — a mapping from signal name to + a dict of arrays keyed by ``'mean'``, ``'std'``, ``'min_val'``, and + ``'max_val'`` — and writes found values into the corresponding + :class:`PreprocessConfig` objects in ``self.signal_configs``. + Signals not present in ``self.preprocessing_stats`` are unchanged. + + Returns + ------- + None + """ for config in self.signal_configs: if config.name in self.preprocessing_stats: stats = self.preprocessing_stats[config.name] @@ -670,14 +1099,30 @@ def _update_preprocessing_stats(self): config.preprocess.max_val = stats["max_val"] def _apply_preprocessing( - self, tensor: torch.Tensor, config: PreprocessConfig + self, + tensor: torch.Tensor, + config: PreprocessConfig ) -> torch.Tensor: - """Apply preprocessing transformation. + """ + Apply the configured preprocessing transformation to a tensor. + + Statistics stored on *config* (mean, std, min_val, max_val) are + reshaped to ``(C, 1, 1)`` or ``(C, 1)`` as needed so they broadcast + correctly over time and frequency dimensions. + + Parameters + ---------- + tensor : torch.Tensor + Input data; either a spectrogram of shape ``(C, F, T)`` or a + time-series of shape ``(C, T)``. + config : PreprocessConfig + Preprocessing configuration specifying ``method`` and the + optional statistical parameters. - Args: - tensor: Can be: - - Spectrogram: (channels, freq_bins, time_frames) - - Timeseries: (channels, 1, time_frames) + Returns + ------- + torch.Tensor + Transformed tensor with the same shape as *tensor*. """ if config.method == "none": return tensor @@ -752,7 +1197,17 @@ def _apply_preprocessing( return tensor def _open_hdf5(self): - """Open HDF5 file for this worker with optimized cache settings.""" + """ + Open the HDF5 file for the current worker, if not already open. + + Uses a large chunk cache (256 MB, 10 000 slots) to amortise + repeated random-access reads during training. The open file handle + is stored in ``self.h5_file`` and reused across subsequent calls. + + Returns + ------- + None + """ if self.h5_file is None: self.h5_file = h5py.File( self.hdf5_path, @@ -887,13 +1342,22 @@ def _load_signal_raw( return tensor def _compute_stft(self, signal: torch.Tensor) -> torch.Tensor: - """Compute STFT magnitude spectrogram. + """ + Compute the STFT magnitude spectrogram of a multi-channel signal. + + Applies a Hann-windowed STFT and discards the DC component (bin 0) + to avoid extreme values from the signal offset. - Args: - signal: (channels, time_samples) at native sampling rate + Parameters + ---------- + signal : torch.Tensor + Multi-channel time-series of shape ``(C, T)`` at the signal's + native sampling rate. - Returns: - Magnitude spectrogram (channels, freq_bins, time_frames) + Returns + ------- + torch.Tensor + Magnitude spectrogram of shape ``(C, n_fft // 2, time_frames)``. """ spec = torch.stft( signal, @@ -906,7 +1370,24 @@ def _compute_stft(self, signal: torch.Tensor) -> torch.Tensor: return torch.abs(spec) def _load_metadata(self, f: h5py.File) -> dict: - """Load text data.""" + """ + Load shot metadata from the HDF5 file. + + Extracts the operator log stored under ``f['log']['data']`` as a + UTF-8 string. Returns an empty string for the ``'text'`` key when + the ``'log'`` group is absent. + + Parameters + ---------- + f : h5py.File + Open HDF5 file handle for the shot. + + Returns + ------- + dict + Dictionary with a single key ``'text'`` mapping to the decoded + log string. + """ metadata = {} # Text @@ -921,7 +1402,17 @@ def _load_metadata(self, f: h5py.File) -> dict: return metadata - def __len__(self): + def __len__(self) -> int: + """ + Return the number of non-overlapping chunks in the shot. + + Returns + ------- + int + ``ceil(duration / chunk_duration_s)`` in standard mode, or + ``floor((duration - prediction_horizon_s) / chunk_duration_s)`` + in prediction mode; at least 1. + """ return self.length def __getstate__(self): @@ -937,15 +1428,26 @@ def __setstate__(self, state): def _process_signal( self, data: torch.Tensor, config: SignalConfig ) -> torch.Tensor: - """Process signal for extended window (input + prediction horizon). + """ + Transpose, optionally compute STFT, and preprocess a raw signal. - Args: - data: Raw signal data - config: Signal configuration + Parameters + ---------- + data : torch.Tensor + Raw signal of shape ``(T, C)`` as returned by + :meth:`_load_signal_raw`. + config : SignalConfig + Configuration for the signal, including ``apply_stft`` and + ``preprocess`` settings. - Returns: - STFT signals: (channels, freq_bins, extended_frames) - Non-STFT signals: (channels, 1, extended_frames) + Returns + ------- + torch.Tensor + Processed tensor: + + - ``(C, n_fft // 2, time_frames)`` when + ``config.apply_stft`` is ``True``. + - ``(C, T)`` otherwise. """ # Step 1: Convert to torch and transpose to (channels, time) tensor = data.T @@ -968,10 +1470,30 @@ def _load_movie_raw( t_start: float, t_end: float ) -> torch.Tensor: - """Load raw movie data without resampling (for prediction mode). + """ + Load, window, and resample a raw movie to the target resolution. + + Reads frame data from the HDF5 file, clips to the requested time + window, and resamples with trilinear interpolation to the target + frame rate and spatial dimensions defined in *config*. - Returns: - Raw movie array at native frame rate, shape (time, height, width) + Parameters + ---------- + f : h5py.File + Open HDF5 file handle for the shot. + config : MovieConfig + Camera configuration specifying target FPS, height, and width. + t_start : float + Start time in seconds (relative to t=0). + t_end : float + End time in seconds (relative to t=0). + + Returns + ------- + torch.Tensor + Resampled movie of shape + ``(round((t_end - t_start) * config.target_fps), + config.height, config.width)``. """ duration_s = t_end - t_start @@ -1073,7 +1595,26 @@ def _load_movie_raw( return tensor - def __getitem__(self, idx): + def __getitem__(self, idx: int) -> dict: + """ + Return the data chunk at position *idx*. + + Opens the HDF5 file on the first call (lazy initialisation) and + delegates to :meth:`_getitem_standard` or + :meth:`_getitem_prediction` depending on ``self.prediction_mode``. + + Parameters + ---------- + idx : int + Chunk index in ``[0, len(self))``. + + Returns + ------- + dict + In standard mode: flat mapping from signal/movie/metadata name + to processed tensor or string. + In prediction mode: ``{'inputs': dict, 'targets': dict}``. + """ self._open_hdf5() if self.prediction_mode: @@ -1081,8 +1622,27 @@ def __getitem__(self, idx): else: return self._getitem_standard(idx) - def _getitem_standard(self, idx): - """Original __getitem__ logic.""" + def _getitem_standard(self, idx: int) -> dict: + """ + Load and return the data chunk at *idx* in standard mode. + + Computes the time window + ``[idx * chunk_duration_s, (idx + 1) * chunk_duration_s]``, loads + all active signals, movies, and metadata, and returns them as a + flat dictionary. + + Parameters + ---------- + idx : int + Chunk index in ``[0, len(self))``. + + Returns + ------- + dict[str, torch.Tensor | str] + Keys are signal/movie names plus ``'text'`` (when ``'text'`` + is in ``self.input_signals``). Tensor shapes follow the rules + in :meth:`_process_signal` and :meth:`_load_movie_raw`. + """ t_start = idx * self.chunk_duration_s t_end = t_start + self.chunk_duration_s @@ -1110,8 +1670,29 @@ def _getitem_standard(self, idx): return {**all_signals, **all_movies, **all_metadata} - def _getitem_prediction(self, idx): - """Load extended window, process jointly, then split into input/target.""" + def _getitem_prediction(self, idx: int) -> dict: + """ + Load an extended window and split it into input and target chunks. + + The extended window spans + ``[idx * chunk_duration_s, + idx * chunk_duration_s + chunk_duration_s + prediction_horizon_s]``. + All configured signals are processed over this window and then split + at ``chunk_duration_s`` frames into the input and target portions. + + Parameters + ---------- + idx : int + Chunk index in ``[0, len(self))``. + + Returns + ------- + dict + ``{'inputs': dict[str, torch.Tensor | str], + 'targets': dict[str, torch.Tensor]}``. + Each inner dict maps signal names to the corresponding slice of + the processed tensor. + """ # Extended window: from t to t + chunk_duration + prediction_horizon t_start = idx * self.chunk_duration_s t_end = t_start + self.chunk_duration_s + self.prediction_horizon_s @@ -1183,7 +1764,16 @@ def _getitem_prediction(self, idx): return {"inputs": inputs, "targets": targets} def __del__(self): - """Close file when dataset is deleted.""" + """ + Close the HDF5 file handle when the dataset is garbage-collected. + + Silently ignores errors that may occur if the file was already + closed or if Python is shutting down. + + Returns + ------- + None + """ if self.h5_file is not None: try: self.h5_file.close() From e7c8c9ca5b1e4d47d752b3016de3b03ab1a391db Mon Sep 17 00:00:00 2001 From: renierts Date: Mon, 2 Mar 2026 16:54:03 -0500 Subject: [PATCH 24/30] A lot of bugfixes in the dataloader and prepare_data.py --- .../data_preparation/make_processing_stats.py | 5 +- scripts/data_preparation/prepare_data.py | 259 +++++++++--------- .../data/data_loader.py | 228 ++++++++------- 3 files changed, 259 insertions(+), 233 deletions(-) diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index 6958b8d..98e836c 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -5,7 +5,7 @@ def main(): hdf5_files = sorted( Path("/scratch/gpfs/EKOLEMEN/foundation_model/" - ).glob("20000[0-7]_processed.h5") + ).glob("2000*_processed.h5") ) # hdf5_files = sorted( @@ -16,7 +16,7 @@ def main(): # STFT spectrograms "mhr", "ece", "co2", # actuators / gas / heating - "gas", "ech", "pin", "tin", "gas_flow", "gas_raw", "ich", + "ech", "pin", "tin", "gas_flow", "gas_raw", "ich", # diagnostics "filterscopes", "vib", "mse", "ts_core_density", "ts_core_temp", "ts_tangential_density", "ts_tangential_temp", "cer_ti", "cer_rot", @@ -32,6 +32,7 @@ def main(): hdf5_path=str(f), input_signals=all_input_signals, target_signals=all_input_signals, + max_duration_s=10., ) for f in hdf5_files] compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') diff --git a/scripts/data_preparation/prepare_data.py b/scripts/data_preparation/prepare_data.py index ac9d979..054f036 100644 --- a/scripts/data_preparation/prepare_data.py +++ b/scripts/data_preparation/prepare_data.py @@ -399,155 +399,160 @@ def resample_signal_groups(loaded_data: dict[str, dict]) -> dict[str, dict]: continue # Handle stacked array (channels x time) - all share same time axis - if isinstance(data, np.ndarray) and time.ndim == 1: + # Standard 1D signals usually come in as (channels, time) + # But we need to be careful not to catch video data here if it happens to match criteria + # checking ndim=2 helps distinguish 1D signals from 3D video tensors + if isinstance(data, np.ndarray) and time.ndim == 1 and data.ndim == 2: if time.size == 0: print(f" Skipping - no time axis") resampled[group_name] = group_data.copy() continue - # Transpose from (channels, time) to (time, channels) - data_transposed = data.T - time = time / 1000 + pass - print(f" Data shape: {data.shape}") - print(f" Time range: {time[0]:.3f} to {time[-1]:.3f} s") - print(f" Target frequency: {target_freq} Hz") + # --- Robust General Processing --- + print(f" Processing signals with potentially different time axes") - # Resample all channels together (they share time axis) - new_time, resampled_data = _resample_time_series( - data_transposed, time, target_freq - ) + # Normalize inputs to lists + if isinstance(data, np.ndarray): + if data.ndim == 2: # (Channels, Time) + data_list = list(data) + else: + # For 3D+ data, it's likely (Channels, ...) + # or if it's a single video volume, maybe it shouldn't be split yet? + # But the loop below expects data_list to match num_channels. + # If shape is (720, 240, 420), this is ONE signal (one channel). + # If data is a list, it's a list of signals. + data_list = [data[i] for i in range(data.shape[0])] + else: + data_list = list(data) - # Transpose back to (channels, time) - resampled_data = resampled_data.T + if isinstance(time, np.ndarray): + # shared time axis + time_list = [time] * len(data_list) + else: + time_list = list(time) - print(f" Resampled: {resampled_data.shape}") - print(f" New time range: {new_time[0]:.3f} " - f"to {new_time[-1]:.3f} s") + # Step 1: Find global time range across ALL signals + t_min = np.inf + t_max = -np.inf - new_time = new_time * 1000 + for t in time_list: + if isinstance(t, np.ndarray) and len(t) > 0: + t_min = min(t_min, t[0] / 1000) + t_max = max(t_max, t[-1] / 1000) + if np.isinf(t_min) or np.isinf(t_max): + print(f" No valid time data found") resampled[group_name] = group_data.copy() - resampled[group_name]['data'] = resampled_data - resampled[group_name]['time'] = new_time + continue - # Handle list of arrays OR stacked with different time axes - else: - print(f" Processing {len(data)} signals " - f"with potentially different time axes") + # Step 2: Create single uniform time grid for entire group + dt = 1.0 / target_freq + n_samples = int(np.ceil((t_max - t_min) / dt)) + 1 + common_time = t_min + np.arange(n_samples) * dt + + print(f" Global time range: {t_min:.3f} to {t_max:.3f} s") + print(f" Common time grid: {len(common_time)} samples @ {target_freq} Hz") + common_time = common_time * 1000 # Convert back to ms for interpolation + + # Step 3: Determine Spatial Shape and Prepare Output Array + spatial_shape = None + + def fix_video_shape(d): + # Force reshape for EDICAM video data if size matches + # The user confirmed that reshaping to (-1, 240, 720) is correct. + # 240*720 = 172800 pixels per frame. + PIXELS_PER_FRAME = 240 * 720 + if d.size > 0 and d.size % PIXELS_PER_FRAME == 0: + frames = d.size // PIXELS_PER_FRAME + # Return shape (Time, Height, Width) + return d.reshape(frames, 240, 720) + return d + + # Scan for shape + for d in data_list: + d_fixed = fix_video_shape(d) + # If it's a video, d_fixed will be (Time, 240, 720) -> ndim=3 + if isinstance(d_fixed, np.ndarray) and d_fixed.ndim > 1 and d_fixed.size > 0: + # Standardize on (Time, H, W) -> Spatial is (H, W) + if d_fixed.ndim == 3: + spatial_shape = d_fixed.shape[1:] + break - # Step 1: Find global time range across ALL signals - # time_list = time if isinstance(time, list) else [time] * len(data) - time_list = time if isinstance(time, list) else list(time) - data_list = data if isinstance(data, list) else list(data) + # Allocate output array: (Channels, Time, H, W) + # This is the PyTorch-friendly format we want to end up with. + if spatial_shape is not None: + resampled_data_array = np.full( + (num_channels, len(common_time)) + spatial_shape, np.nan, dtype='f4') + else: + resampled_data_array = np.full((num_channels, len(common_time)), np.nan, + dtype='f4') + + # Step 4: Resample + for i, (signal_data, signal_time) in enumerate(zip(data_list, time_list)): + if i >= num_channels: break + + signal_data = fix_video_shape(signal_data) + + if not isinstance(signal_data, np.ndarray) or signal_data.size == 0: continue + if not isinstance(signal_time, np.ndarray) or signal_time.size == 0: continue + + if len(signal_time) < 2: continue + + # --- 1D Case --- + if signal_data.ndim == 1: + valid_mask = ~np.isnan(signal_data) + if np.sum(valid_mask) >= 2: + f = interp1d(signal_time[valid_mask], signal_data[valid_mask], + kind='linear', bounds_error=False, fill_value=np.nan) + resampled_data_array[i, :] = f(common_time) + + # --- Video / Multi-dim Case --- + # We now expect (Time, H, W) from fix_video_shape + elif signal_data.ndim == 3: + # signal_data is (T, H, W) + # We need to interpolate along axis 0 (Time) + + # Check if time dimension matches signal_time length + if signal_data.shape[0] != len(signal_time): + print( + f" Warning: Time dim {signal_data.shape[0]} != Time vec {len(signal_time)}") + # Try to transpose if it helps (e.g. if it came in as H,W,T) + if signal_data.shape[-1] == len(signal_time): + signal_data = np.moveaxis(signal_data, -1, 0) + else: + continue - t_min = np.inf - t_max = -np.inf + T_in, H, W = signal_data.shape - for t in time_list: - if isinstance(t, np.ndarray) and len(t) > 0: - t_min = min(t_min, t[0] / 1000) - t_max = max(t_max, t[-1] / 1000) + # Flatten spatial dims: (T, H*W) + flat_data = signal_data.reshape(T_in, -1) - if np.isinf(t_min) or np.isinf(t_max): - print(f" No valid time data found") - resampled[group_name] = group_data.copy() - continue + # Interpolate along axis 0 + f = interp1d(signal_time, flat_data, axis=0, kind='linear', + bounds_error=False, fill_value=np.nan) - # Step 2: Create single uniform time grid for entire group - dt = 1.0 / target_freq - n_samples = int(np.ceil((t_max - t_min) / dt)) + 1 - common_time = t_min + np.arange(n_samples) * dt - - print(f" Global time range: {t_min:.3f} to {t_max:.3f} s") - print(f" Common time grid: {len(common_time)} " - f"samples @ {target_freq} Hz") - common_time = common_time * 1000 - - # Step 3: Resample each signal to the COMMON time grid - # Detect spatial dimensions from the first non-empty multi-dim channel. - # For video the shape is (W, H, T) so spatial_shape = (W, H); - # for 1D time series spatial_shape stays None. - spatial_shape = None - for d in data_list: - if (isinstance(d, np.ndarray) and d.ndim > 1 - and d.size > 0): - spatial_shape = d.shape[:-1] # all axes except last (time) - break + flat_resampled = f(common_time) - if spatial_shape is not None: - resampled_data_array = np.full( - (num_channels,) + spatial_shape + (len(common_time),), - np.nan, dtype='f8') - else: - resampled_data_array = np.full( - (num_channels, len(common_time)), np.nan, dtype='f8') + # Reshape back to (NewTime, H, W) + resampled_nd = flat_resampled.reshape(len(common_time), H, W) - for i, (signal_data, signal_time) in enumerate( - zip(data_list, time_list)): - if i >= num_channels: - break + # Assign to output array (Channels, Time, H, W) + # Since resampled_data_array is (C, T, H, W), we assign directly + try: + resampled_data_array[i] = resampled_nd + except ValueError: + print( + f" Mismatch: Target {resampled_data_array[i].shape}, Got {resampled_nd.shape}") - if (not isinstance(signal_data, np.ndarray) - or signal_data.size == 0): - continue # Leave as NaN - - if (not isinstance(signal_time, np.ndarray) - or signal_time.size == 0): - continue # Leave as NaN - - if signal_data.ndim == 1: - # 1D time series: interpolate directly - valid_mask = ~np.isnan(signal_data) - if np.sum(valid_mask) >= 2: - interpolator = interp1d( - signal_time[valid_mask], - signal_data[valid_mask], - kind='linear', - bounds_error=False, - fill_value=np.nan - ) - resampled_data_array[i, :] = interpolator(common_time) - else: - # Multi-dim channel (e.g. video shape (W, H, T)): - # time is the last axis; interpolate per spatial location. - ch_spatial = signal_data.shape[:-1] - n_time = signal_data.shape[-1] - - # (spatial..., T) -> (T, spatial_flat) - data_t = np.moveaxis(signal_data, -1, 0) - data_flat = data_t.reshape(n_time, -1) - - resampled_flat = np.full( - (len(common_time), data_flat.shape[1]), - np.nan, dtype='f8') - - for j in range(data_flat.shape[1]): - pixel_series = data_flat[:, j] - valid_mask = ~np.isnan(pixel_series) - if np.sum(valid_mask) >= 2: - interpolator = interp1d( - signal_time[valid_mask], - pixel_series[valid_mask], - kind='linear', - bounds_error=False, - fill_value=np.nan - ) - resampled_flat[:, j] = interpolator(common_time) - - # (new_T, spatial_flat) -> (spatial..., new_T) - resampled_nd = resampled_flat.reshape( - (len(common_time),) + ch_spatial) - resampled_data_array[i] = np.moveaxis(resampled_nd, 0, -1) - - valid_samples = int(np.sum(~np.isnan(resampled_data_array[i]))) - print(f" Channel {i}: {valid_samples} valid samples") + valid_samples = int(np.sum(~np.isnan(resampled_data_array[i]))) + print(f" Channel {i}: {valid_samples} valid samples") - resampled[group_name] = group_data.copy() - resampled[group_name]['data'] = resampled_data_array - resampled[group_name]['time'] = common_time / 1000. - print( - f" Resampled to common grid: {resampled_data_array.shape}") + resampled[group_name] = group_data.copy() + resampled[group_name]['data'] = resampled_data_array + resampled[group_name]['time'] = common_time / 1000.0 + print(f" Final group shape: {resampled_data_array.shape}") return resampled diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index 6a0359b..9297be5 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -291,7 +291,6 @@ def compute(self): def compute_preprocessing_stats( datasets: "list[TokamakH5Dataset]", output_path: str | Path = "preprocessing_stats.pt", - num_samples: int = 1000, batch_size: int = 32, num_workers: int = 1, ) -> dict[str, dict[str, np.ndarray]]: @@ -299,11 +298,10 @@ def compute_preprocessing_stats( Compute per-modality preprocessing statistics over a collection of datasets. - For each dataset, draws a random subset of up to *num_samples* chunks, - concatenates the subsets, then accumulates running statistics with - :class:`WelfordTensor`. The result is saved to *output_path* via - :func:`torch.save`. Only modalities that actually appear in the loaded - batches are included in the output. + Iterates over all chunks in every dataset, accumulates running statistics + with :class:`WelfordTensor`, and saves the result to *output_path* via + :func:`torch.save`. Only modalities that appear in the loaded batches + are included in the output. Parameters ---------- @@ -313,9 +311,6 @@ def compute_preprocessing_stats( output_path : str or Path, optional Filesystem path for the saved ``.pt`` statistics file. Default is ``"preprocessing_stats.pt"``. - num_samples : int, optional - Maximum number of chunks to draw randomly from *each* dataset. - Default is ``1000``. batch_size : int, optional Batch size for the internal DataLoader. Default is ``32``. num_workers : int, optional @@ -336,32 +331,31 @@ def compute_preprocessing_stats( ``'max_val'`` Per-channel maximum, shape ``(C,)``. """ - from torch.utils.data import ConcatDataset, Subset + from torch.utils.data import ConcatDataset from tqdm import tqdm - # Draw a random subset from each dataset to stay within num_samples - sampled = [] - for ds in datasets: - n = min(num_samples, len(ds)) - indices = torch.randperm(len(ds))[:n].tolist() - sampled.append(Subset(ds, indices)) - - combined = ConcatDataset(sampled) + combined = ConcatDataset(datasets) dataloader = DataLoader( combined, batch_size=batch_size, collate_fn=collate_fn, num_workers=num_workers) - # Use instance-level configs (deep copies that may have been modified) + # Use instance-level configs (deep copies that may have been modified). signal_configs = datasets[0].signal_configs movie_configs = datasets[0].movie_configs welford_stats = { - cfg.name: WelfordTensor() for cfg in signal_configs + movie_configs} + cfg.name: WelfordTensor() + for cfg in signal_configs + movie_configs} for batch in tqdm(dataloader): for modality_name, tensor in batch.items(): if modality_name not in welford_stats: continue + # Movies arrive as (B, C, T, H, W); flatten spatial/temporal dims + # to (B, C, T*H*W) so WelfordTensor computes per-channel stats. + if tensor.ndim == 5: + B, C, T, H, W = tensor.shape + tensor = tensor.reshape(B, C, T * H * W) welford_stats[modality_name].update(tensor) # Only include trackers that received data @@ -445,16 +439,20 @@ class SignalConfig: Ordered list of HDF5 group paths to search for the signal data. The first path that exists in the file is used. num_channels : int - Expected number of signal channels (``C``). + Number of output channels after applying *channels_to_use*. Must + equal ``len(range(*channels_to_use.indices(N)))`` when + *channels_to_use* is not ``None``. target_fs : float Target sampling frequency in Hz. The raw signal is resampled to this rate before being returned. apply_stft : bool If ``True``, compute an STFT magnitude spectrogram after loading, yielding output shape ``(C, F, T)``. If ``False``, the signal is - returned as ``(C, 1, T)``. - channels_to_use : slice - Optional slice to select specific channels + returned as ``(C, T)``. + channels_to_use : slice or None, optional + Slice applied to the HDF5 channel axis before writing to the output + buffer. ``None`` (default) passes all available channels through, + truncating or zero-padding to *num_channels* as needed. preprocess : PreprocessConfig, optional Preprocessing transformation applied after the STFT (or pass-through). Defaults to :class:`PreprocessConfig` with @@ -466,7 +464,7 @@ class SignalConfig: num_channels: int target_fs: float apply_stft: bool - channels_to_use: slice = field(default_factory=lambda: slice(0, -1)) + channels_to_use: Optional[slice] = None preprocess: PreprocessConfig = None def __post_init__(self): @@ -552,6 +550,8 @@ class TokamakH5Dataset(Dataset): data-preparation pipeline). chunk_duration_s : float, optional Duration of each time window in seconds. Default is ``0.5``. + max_duration_s : float, optional + Maximum duration of a shot to be considered. n_fft : int, optional FFT size used for STFT computation. Determines the number of frequency bins: ``n_fft // 2 + 1``. Default is ``1024``. @@ -609,11 +609,10 @@ class TokamakH5Dataset(Dataset): ========================== ======== ========== ===== ================== Name Channels Target fs STFT Preprocessing ========================== ======== ========== ===== ================== - ``mhr`` 8 500 kHz yes log - ``ece`` 48 500 kHz yes log + ``mhr`` 6 500 kHz yes log + ``ece`` 40 500 kHz yes log ``co2`` 4 500 kHz yes log - ``gas`` 5 10 kHz no none - ``ech`` 11 10 kHz no none + ``ech`` 12 10 kHz no none ``pin`` 8 10 kHz no standardize ``tin`` 8 10 kHz no none ``mse`` 69 100 Hz no none @@ -652,19 +651,19 @@ class TokamakH5Dataset(Dataset): SignalConfig( "mhr", ["mhr"], - 8, + 6, 500e3, apply_stft=True, - channels_to_use=slice(2, 8), # Use only the first 8 channels + channels_to_use=slice(2, 8), # Skip first 2 channels preprocess=PreprocessConfig(method="log"), ), SignalConfig( "ece", ["ece"], - 48, + 40, 500e3, apply_stft=True, - channels_to_use=slice(0, 40), # Use only the first 40 channels + channels_to_use=slice(0, 40), # Use the first 40 of 48 channels preprocess=PreprocessConfig(method="log"), ), SignalConfig( @@ -675,25 +674,17 @@ class TokamakH5Dataset(Dataset): apply_stft=True, preprocess=PreprocessConfig(method="log"), ), - SignalConfig( - "gas", - ["gas"], - 5, - 10e3, - apply_stft=False, - preprocess=PreprocessConfig(method="none"), - ), SignalConfig( "ech", ["ech"], - 11, + 12, 10e3, apply_stft=False, preprocess=PreprocessConfig(method="none"), ), SignalConfig( "pin", - ["pin"], + ["pinj"], 8, 10e3, apply_stft=False, @@ -701,7 +692,7 @@ class TokamakH5Dataset(Dataset): ), SignalConfig( "tin", - ["tin"], + ["tinj"], 8, 10e3, apply_stft=False, @@ -863,14 +854,15 @@ class TokamakH5Dataset(Dataset): ] MOVIE_CONFIGS = [ - MovieConfig("irtv", ["irtv"], 1, 50, 513, 640), - MovieConfig("tangtv", ["tangtv"], 1, 50, 240, 720), + MovieConfig("irtv", ["irtv"], 6, 50, 513, 640), + MovieConfig("tangtv", ["tangtv"], 7, 50, 240, 720), ] def __init__( self, hdf5_path: str | Path, chunk_duration_s: float = 0.5, + max_duration_s: float = 12.0, n_fft: int = 1024, hop_length: int = 256, preprocessing_stats: Optional[dict] = None, @@ -907,7 +899,7 @@ def __init__( try: with h5py.File(self.hdf5_path, "r") as f: self.duration, self.t0_indices = \ - self._compute_duration_and_t0_indices(f) + self._compute_duration_and_t0_indices(f, max_duration_s) except OSError as e: print(self.hdf5_path) raise e @@ -973,7 +965,8 @@ def _find_t0_index(self, xdata_ms: np.ndarray) -> tuple[int, float]: def _compute_duration_and_t0_indices( self, - f: h5py.File + f: h5py.File, + max_duration_s: float | None = None, ) -> tuple[float, dict]: """ Compute shot duration from t=0 and locate the t=0 index per signal. @@ -1031,7 +1024,9 @@ def _compute_duration_and_t0_indices( # Duration from t=0 to end duration_s = (xdata_ms[-1] - 0.0) / 1000.0 - max_duration = max(max_duration, duration_s) + max_duration = max( + max_duration, min(duration_s, max_duration_s) + ) break @@ -1063,7 +1058,9 @@ def _compute_duration_and_t0_indices( } duration_s = (xdata_ms[-1] - 0.0) / 1000.0 - max_duration = max(max_duration, duration_s) + max_duration = max( + max_duration, min(max_duration_s, duration_s) + ) break @@ -1113,8 +1110,11 @@ def _apply_preprocessing( Parameters ---------- tensor : torch.Tensor - Input data; either a spectrogram of shape ``(C, F, T)`` or a - time-series of shape ``(C, T)``. + Input data; one of: + + - spectrogram ``(C, F, T)`` + - time-series ``(C, T)`` + - video ``(C, T, H, W)`` config : PreprocessConfig Preprocessing configuration specifying ``method`` and the optional statistical parameters. @@ -1127,14 +1127,16 @@ def _apply_preprocessing( if config.method == "none": return tensor - # Determine how to reshape statistics based on tensor dimensions - # For (C, F, T) spectrograms, we want (C, 1, 1) for per-channel stats - # For (C, 1, T) timeseries, we want (C, 1, 1) for per-channel stats - if tensor.ndim == 3: - # Reshape to (channels, 1, 1) for proper broadcasting + # Reshape per-channel statistics for correct broadcasting. + # Stats have shape (C,); we add trailing singleton dims to match ndim. + if tensor.ndim == 4: + # (C, T, H, W) — video + reshape_dims = (tensor.shape[0], 1, 1, 1) + elif tensor.ndim == 3: + # (C, F, T) — spectrogram reshape_dims = (tensor.shape[0], 1, 1) elif tensor.ndim == 2: - # Reshape to (channels, 1) + # (C, T) — time-series reshape_dims = (tensor.shape[0], 1) else: reshape_dims = None @@ -1266,8 +1268,9 @@ def _load_signal_raw( xdata_ds = data_group["xdata"] # Get time range and sample count - xdata_start_s = xdata_ds[0] / 1000.0 - xdata_end_s = xdata_ds[-1] / 1000.0 + xdata_start_s = xdata_ds[0] + xdata_end_s = xdata_ds[-1] + n_samples = xdata_ds.shape[0] if n_samples < 2 or xdata_end_s == xdata_start_s: @@ -1296,7 +1299,7 @@ def _load_signal_raw( # Step 3: Load data if there's any overlap if hdf5_start_clamped < hdf5_end_clamped: - data = ydata_ds[hdf5_start_clamped:hdf5_end_clamped] + data = ydata_ds[:, hdf5_start_clamped:hdf5_end_clamped].T np.nan_to_num(data, copy=False, nan=0.0) # Step 4: Calculate where to insert in output array @@ -1318,12 +1321,18 @@ def _load_signal_raw( # Insert data into output if src_start < src_end and output_start < output_end: - if data.shape[1] == config.num_channels: - output[output_start:output_end] = data[src_start:src_end] - elif data.shape[1] > config.num_channels: - output[output_start:output_end] = data[src_start:src_end, :config.num_channels] + chunk = data[src_start:src_end] + + # Apply channel selection if specified + if config.channels_to_use is not None: + chunk = chunk[:, config.channels_to_use] + + if chunk.shape[1] == config.num_channels: + output[output_start:output_end] = chunk + elif chunk.shape[1] > config.num_channels: + output[output_start:output_end] = chunk[:, :config.num_channels] else: - output[output_start:output_end, :data.shape[1]] = data[src_start:src_end] + output[output_start:output_end, :chunk.shape[1]] = chunk # Step 6: Convert to tensor and resample to target frequency tensor = torch.from_numpy(output).float() @@ -1473,9 +1482,10 @@ def _load_movie_raw( """ Load, window, and resample a raw movie to the target resolution. - Reads frame data from the HDF5 file, clips to the requested time - window, and resamples with trilinear interpolation to the target - frame rate and spatial dimensions defined in *config*. + Reads frame data from the HDF5 file (stored as ``(C, W, H, T)``), + clips to the requested time window, collapses channels via + ``nanmean``, and resamples with trilinear interpolation to the + target frame rate and spatial dimensions defined in *config*. Parameters ---------- @@ -1492,7 +1502,8 @@ def _load_movie_raw( ------- torch.Tensor Resampled movie of shape - ``(round((t_end - t_start) * config.target_fps), + ``(config.channels, + round((t_end - t_start) * config.target_fps), config.height, config.width)``. """ duration_s = t_end - t_start @@ -1510,33 +1521,44 @@ def _load_movie_raw( except KeyError: continue + if data_group is None: + return torch.zeros( + (config.channels, round(duration_s * config.target_fps), + config.height, config.width) + ) + ydata_ds = data_group["ydata"] xdata_ds = data_group["xdata"] if ydata_ds.size == 0: return torch.zeros( - (round(duration_s * config.target_fps), config.height, config.width) + (config.channels, round(duration_s * config.target_fps), + config.height, config.width) ) # Get time range and frame count - xdata_start_s = xdata_ds[0] / 1000.0 - xdata_end_s = xdata_ds[-1] / 1000.0 + xdata_start_s = xdata_ds[0] + xdata_end_s = xdata_ds[-1] n_frames = xdata_ds.shape[0] if n_frames < 2 or xdata_end_s == xdata_start_s: return torch.zeros( - (round(duration_s * config.target_fps), config.height, config.width) + (config.channels, round(duration_s * config.target_fps), + config.height, config.width) ) # Compute actual frame rate from the data actual_fps = (n_frames - 1) / (xdata_end_s - xdata_start_s) - # Get actual dimensions from data - raw_height, raw_width = ydata_ds.shape[1], ydata_ds.shape[2] + # ydata layout: (C, W, H, T) — time is the last axis + raw_channels = ydata_ds.shape[0] + raw_height = ydata_ds.shape[2] # H + raw_width = ydata_ds.shape[3] # W # Step 1: Initialize output array with zeros at actual fps + # (T, C, H, W) output = np.zeros( - (round(duration_s * actual_fps), raw_height, raw_width), + (raw_channels, round(duration_s * actual_fps), raw_height, raw_width), dtype=np.float32 ) @@ -1552,45 +1574,43 @@ def _load_movie_raw( # Step 3: Load data if there's any overlap if hdf5_start_clamped < hdf5_end_clamped: - data = ydata_ds[hdf5_start_clamped:hdf5_end_clamped] - data[np.isnan(data)] = 0.0 + chunk = ydata_ds[:, hdf5_start_clamped:hdf5_end_clamped, :, :] + data = np.nan_to_num(chunk, nan=0.0) # Step 4: Calculate where to insert in output array # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fps # This corresponds to output index: (that_time - t_start) * actual_fps output_start = hdf5_start_clamped - hdf5_start - output_end = output_start + data.shape[0] + output_end = output_start + data.shape[1] # Clamp to output bounds src_start = 0 - src_end = data.shape[0] + src_end = data.shape[1] if output_start < 0: src_start = -output_start output_start = 0 - if output_end > output.shape[0]: - src_end -= output_end - output.shape[0] - output_end = output.shape[0] + if output_end > output.shape[1]: + src_end -= output_end - output.shape[1] + output_end = output.shape[1] # Insert data into output if src_start < src_end and output_start < output_end: - output[output_start:output_end] = data[src_start:src_end] + output[:, output_start:output_end] = data[:, src_start:src_end] # Step 5: Convert to tensor and resample to target fps and dimensions tensor = torch.from_numpy(output).float() - # Resample using trilinear interpolation - # Input: (time, height, width) → add batch and channel dims - # Output: (batch=1, channels=1, time, height, width) + # Resample using trilinear interpolation. + # (C, T, H, W) → (1, C, T, H, W) + # → interpolate → (1, C, T', H', W') → (C, T', H', W') tensor = ( - F.interpolate(tensor.unsqueeze(0).unsqueeze(0), - size=(round(duration_s * config.target_fps), - config.height, - config.width, - ), - mode="trilinear", - align_corners=False, - ).squeeze(0).squeeze(0) + F.interpolate( + tensor.unsqueeze(0), # (1, C, T, H, W) + size=(round(duration_s * config.target_fps), config.height, config.width), + mode="trilinear", + align_corners=False, + ).squeeze(0) # (C, T', H', W') ) return tensor @@ -1660,7 +1680,8 @@ def _getitem_standard(self, idx: int) -> dict: raw_movie = self._load_movie_raw( self.h5_file, movie_config, t_start, t_end ) - all_movies[movie_config.name] = raw_movie + all_movies[movie_config.name] = self._apply_preprocessing( + raw_movie, movie_config.preprocess) # Load metadata if "text" in self.input_signals: @@ -1712,9 +1733,9 @@ def _getitem_prediction(self, idx: int) -> dict: for movie_config in self.movie_configs: if movie_config.name not in signals_to_load: continue - # Load raw movie data raw_movie = self._load_movie_raw(self.h5_file, movie_config, t_start, t_end) - all_movies[movie_config.name] = raw_movie + all_movies[movie_config.name] = self._apply_preprocessing( + raw_movie, movie_config.preprocess) # Load metadata all_metadata = self._load_metadata(self.h5_file) @@ -1742,20 +1763,19 @@ def _getitem_prediction(self, idx: int) -> dict: if config.name in self.target_signals: targets[config.name] = signal[..., n_training_frames:] - # Movies: split along time dimension + # Movies: split along the time dimension (dim 1 of (C, T, H, W)) for movie_config in self.movie_configs: if movie_config.name not in signals_to_load: continue movie_name = movie_config.name movie_data = all_movies[movie_name] n_training_frames = round(self.chunk_duration_s * movie_config.target_fps) - # movie_data shape: (extended_movie_frames, height, width) + # movie_data shape: (C, extended_movie_frames, height, width) if movie_name in self.input_signals: - inputs[movie_name] = movie_data[:n_training_frames] + inputs[movie_name] = movie_data[:, :n_training_frames] - # Include movies in targets if specified if movie_name in self.target_signals: - targets[movie_name] = movie_data[n_training_frames:] + targets[movie_name] = movie_data[:, n_training_frames:] # Metadata (text) only goes to inputs if "text" in self.input_signals: From 060a149037259a9923cfb99dc9a824451503a4dd Mon Sep 17 00:00:00 2001 From: renierts Date: Wed, 4 Mar 2026 10:08:34 -0500 Subject: [PATCH 25/30] Many bugfixees in the dataset class and for computing preprocessing stats. This is still not efficient enough and causes memory issues. --- .../data_preparation/make_processing_stats.py | 3 +- scripts/data_preparation/prepare_data.py | 15 +- scripts/slurm/make_processing_stats.sh | 8 +- scripts/slurm/prepare_data.sh | 2 +- .../data/config/modalities/modalities.yaml | 2 +- .../data/data_loader.py | 145 ++++-------------- .../models/model_factory.py | 1 - 7 files changed, 49 insertions(+), 127 deletions(-) diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index 4d47ab9..9bed2d6 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -4,8 +4,7 @@ def main(): hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/foundation_model/" - ).glob("*_processed.h5") + Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("*_processed.h5") ) all_input_signals = [ diff --git a/scripts/data_preparation/prepare_data.py b/scripts/data_preparation/prepare_data.py index 054f036..8b3ba34 100644 --- a/scripts/data_preparation/prepare_data.py +++ b/scripts/data_preparation/prepare_data.py @@ -400,8 +400,9 @@ def resample_signal_groups(loaded_data: dict[str, dict]) -> dict[str, dict]: # Handle stacked array (channels x time) - all share same time axis # Standard 1D signals usually come in as (channels, time) - # But we need to be careful not to catch video data here if it happens to match criteria - # checking ndim=2 helps distinguish 1D signals from 3D video tensors + # But we need to be careful not to catch video data here if it happens + # to match criteria checking ndim=2 helps distinguish 1D signals from + # 3D video tensors if isinstance(data, np.ndarray) and time.ndim == 1 and data.ndim == 2: if time.size == 0: print(f" Skipping - no time axis") @@ -419,9 +420,10 @@ def resample_signal_groups(loaded_data: dict[str, dict]) -> dict[str, dict]: data_list = list(data) else: # For 3D+ data, it's likely (Channels, ...) - # or if it's a single video volume, maybe it shouldn't be split yet? + # or if it's a single video volume, maybe it shouldn't be split + # yet? # But the loop below expects data_list to match num_channels. - # If shape is (720, 240, 420), this is ONE signal (one channel). + # If shape is (W, H, T), this is ONE signal (one channel). # If data is a list, it's a list of signals. data_list = [data[i] for i in range(data.shape[0])] else: @@ -453,8 +455,9 @@ def resample_signal_groups(loaded_data: dict[str, dict]) -> dict[str, dict]: common_time = t_min + np.arange(n_samples) * dt print(f" Global time range: {t_min:.3f} to {t_max:.3f} s") - print(f" Common time grid: {len(common_time)} samples @ {target_freq} Hz") - common_time = common_time * 1000 # Convert back to ms for interpolation + print(f" Common time grid: {len(common_time)} samples " + f"@ {target_freq} Hz") + common_time = common_time * 1000 # Back to ms for interpolation # Step 3: Determine Spatial Shape and Prepare Output Array spatial_shape = None diff --git a/scripts/slurm/make_processing_stats.sh b/scripts/slurm/make_processing_stats.sh index 551164d..f479ea6 100755 --- a/scripts/slurm/make_processing_stats.sh +++ b/scripts/slurm/make_processing_stats.sh @@ -2,11 +2,11 @@ #SBATCH --job-name=make_processing_stats #SBATCH --output=logs/make_processing_stats.out #SBATCH --error=logs/make_processing_stats.err -#SBATCH --cpus-per-task=32 +#SBATCH --cpus-per-task=2 #SBATCH --nodes=1 -#SBATCH --mem-per-cpu=16G -#SBATCH --time=02:00:00 +#SBATCH --mem-per-cpu=64G +#SBATCH --time=24:00:00 #SBATCH --mail-type=all #SBATCH --mail-user=ps9551@princeton.edu -pixi run python ../data_preparation/make_processing_stats.py +pixi run python -u ../data_preparation/make_processing_stats.py diff --git a/scripts/slurm/prepare_data.sh b/scripts/slurm/prepare_data.sh index 1f1ac81..3c9ce28 100755 --- a/scripts/slurm/prepare_data.sh +++ b/scripts/slurm/prepare_data.sh @@ -9,4 +9,4 @@ #SBATCH --mail-type=all # send email on job start, end and fault #SBATCH --mail-user=ps9551@princeton.edu -pixi run python scripts/prepare_data.py +pixi run python -u ../data_preparation/prepare_data.py diff --git a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml index 9b6e0f2..6beba85 100644 --- a/src/tokamak_foundation_model/data/config/modalities/modalities.yaml +++ b/src/tokamak_foundation_model/data/config/modalities/modalities.yaml @@ -4,7 +4,7 @@ input_data_path: /scratch/gpfs/EKOLEMEN/big_d3d_data/d3d_time_series_data output_data_path: /scratch/gpfs/EKOLEMEN/foundation_model -num_workers: 1 +num_workers: 32 signals: filterscopes: diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index caf7987..ca70f78 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -291,8 +291,7 @@ def compute(self): def compute_preprocessing_stats( datasets: "list[TokamakH5Dataset]", output_path: str | Path = "preprocessing_stats.pt", - batch_size: int = 32, - num_workers: int = 1, + batch_size: int = 1, ) -> dict[str, dict[str, np.ndarray]]: """ Compute per-modality preprocessing statistics over a collection of @@ -312,9 +311,7 @@ def compute_preprocessing_stats( Filesystem path for the saved ``.pt`` statistics file. Default is ``"preprocessing_stats.pt"``. batch_size : int, optional - Batch size for the internal DataLoader. Default is ``32``. - num_workers : int, optional - Number of DataLoader worker processes. Default is ``1``. + Batch size for the internal DataLoader. Default is ``1``. Returns ------- @@ -331,14 +328,8 @@ def compute_preprocessing_stats( ``'max_val'`` Per-channel maximum, shape ``(C,)``. """ - from torch.utils.data import ConcatDataset from tqdm import tqdm - combined = ConcatDataset(datasets) - dataloader = DataLoader( - combined, batch_size=batch_size, collate_fn=collate_fn, - num_workers=num_workers) - # Use instance-level configs (deep copies that may have been modified). signal_configs = datasets[0].signal_configs movie_configs = datasets[0].movie_configs @@ -347,16 +338,28 @@ def compute_preprocessing_stats( cfg.name: WelfordTensor() for cfg in signal_configs + movie_configs} - for batch in tqdm(dataloader): - for modality_name, tensor in batch.items(): - if modality_name not in welford_stats: - continue - # Movies arrive as (B, C, T, H, W); flatten spatial/temporal dims - # to (B, C, T*H*W) so WelfordTensor computes per-channel stats. - if tensor.ndim == 5: - B, C, T, H, W = tensor.shape - tensor = tensor.reshape(B, C, T * H * W) - welford_stats[modality_name].update(tensor) + # Iterate one dataset at a time and close each file handle after use. + # Using ConcatDataset + persistent_workers causes all HDF5 file handles + # (each with a 16 MB chunk cache) to accumulate in the worker process, + # exhausting memory after ~1000 files. + for dataset in tqdm(datasets, desc="Files"): + dataloader = DataLoader( + dataset, batch_size=batch_size, collate_fn=collate_fn, + num_workers=0) + for batch in dataloader: + for modality_name, tensor in batch.items(): + if modality_name not in welford_stats: + continue + # Movies arrive as (B, C, T, H, W); flatten spatial/temporal dims + # to (B, C, T*H*W) so WelfordTensor computes per-channel stats. + if tensor.ndim == 5: + B, C, T, H, W = tensor.shape + tensor = tensor.reshape(B, C, T * H * W) + welford_stats[modality_name].update(tensor) + # Explicitly close the HDF5 file handle to free memory before next file. + if dataset.h5_file is not None: + dataset.h5_file.close() + dataset.h5_file = None # Only include trackers that received data final_stats = { @@ -596,10 +599,6 @@ class TokamakH5Dataset(Dataset): duration : float Total shot duration from t = 0 in seconds, as inferred from the HDF5 time axes. - t0_indices : dict - Mapping ``{modality_name: {'index': int, 'time_s': float}}`` - giving the HDF5 array index and exact timestamp (seconds) of - t = 0 for each modality. length : int Number of non-overlapping chunks available (i.e. ``__len__``). n_freq_bins : int @@ -862,12 +861,12 @@ class TokamakH5Dataset(Dataset): ] MOVIE_CONFIGS = [ - MovieConfig("irtv", ["irtv"], 6, 50, 513, 640), + MovieConfig("irtv", ["irtv"], 7, 50, 513, 640), MovieConfig("tangtv", ["tangtv"], 7, 50, 240, 720), ] VALUE_CONFIG = ValueConfig( - rdcc_nbytes=1024**2 * 256, # 256 MB chunk cache + rdcc_nbytes=1024**2 * 16, # 16 MB chunk cache rdcc_nslots=10000, # Number of chunk slots ms_to_s=1/1000, # Conversion factor from milliseconds to seconds ) @@ -912,8 +911,7 @@ def __init__( self.h5_file = None try: with h5py.File(self.hdf5_path, "r") as f: - self.duration, self.t0_indices = \ - self._compute_duration_and_t0_indices(f, max_duration_s) + self.duration = self._compute_duration(f, max_duration_s) except OSError as e: print(self.hdf5_path) raise e @@ -930,65 +928,17 @@ def __init__( self.n_freq_bins = n_fft // 2 + 1 self.stft_window = torch.hann_window(n_fft) - def _find_t0_index(self, xdata_ms: np.ndarray) -> tuple[int, float]: - """ - Find the index and exact time of t=0 in xdata. - - Parameters - ---------- - xdata_ms : np.ndarray - Array of timestamps in milliseconds, assumed sorted ascending. - - Returns - ------- - index : int - Index closest to t=0, or ``-1`` if all data is before t=0. - actual_time_ms : float - The actual timestamp at that index, in milliseconds. - """ - if len(xdata_ms) == 0: - return -1, 0.0 - - if len(xdata_ms) == 1: - # Single sample - use it if >= 0, else -1 - if xdata_ms[0] >= 0: - return 0, xdata_ms[0] - else: - return -1, xdata_ms[0] - - # All data before t=0 - if xdata_ms[-1] < 0: - return -1, xdata_ms[-1] - - # All data after t=0 (first sample is already past t=0) - if xdata_ms[0] > 0: - return 0, xdata_ms[0] - - # t=0 is within range - find nearest index using binary search - idx = np.searchsorted(xdata_ms, 0) - - # searchsorted returns insertion point - # Check if previous index is closer to 0 - if idx > 0 and idx < len(xdata_ms): - if abs(xdata_ms[idx - 1]) < abs(xdata_ms[idx]): - idx = idx - 1 - elif idx >= len(xdata_ms): - idx = len(xdata_ms) - 1 - - return idx, xdata_ms[idx] - - def _compute_duration_and_t0_indices( + def _compute_duration( self, f: h5py.File, max_duration_s: float | None = None, - ) -> tuple[float, dict]: + ) -> float: """ - Compute shot duration from t=0 and locate the t=0 index per signal. + Compute shot duration from t=0. Iterates over all signal and movie configurations, reads the - ``xdata`` timestamps from the HDF5 file, finds the first sample at - or after t=0, and accumulates the maximum duration across all - available diagnostics. + ``xdata`` timestamps from the HDF5 file, and accumulates the + maximum duration across all available diagnostics. Parameters ---------- @@ -1000,14 +950,8 @@ def _compute_duration_and_t0_indices( max_duration : float Duration in seconds from t=0 to the last sample, across all signals and movies. Guaranteed to be at least 1.0 s. - t0_indices : dict[str, dict[str, int | float]] - Mapping from signal/movie name to a dict with keys: - - - ``'index'``: first HDF5 sample index where ``xdata >= 0``. - - ``'time_s'``: actual timestamp at that index, in seconds. """ max_duration = 0.0 - t0_indices = {} # Process signals for config in self.signal_configs: @@ -1023,19 +967,6 @@ def _compute_duration_and_t0_indices( if len(xdata_ms) < 2: continue - # Find first index where t >= 0 - t0_idx = np.searchsorted(xdata_ms, 0, side="left") - - # If all data is before t=0, skip - if t0_idx >= len(xdata_ms): - continue - - # Store both index and actual time at that index - t0_indices[config.name] = { - "index": int(t0_idx), - "time_s": float(xdata_ms[t0_idx]) / 1000.0, - } - # Duration from t=0 to end duration_s = (xdata_ms[-1] - 0.0) / 1000.0 max_duration = max( @@ -1061,16 +992,6 @@ def _compute_duration_and_t0_indices( if len(xdata_ms) < 2: continue - t0_idx = np.searchsorted(xdata_ms, 0, side="left") - - if t0_idx >= len(xdata_ms): - continue - - t0_indices[movie_config.name] = { - "index": int(t0_idx), - "time_s": float(xdata_ms[t0_idx]) / 1000.0, - } - duration_s = (xdata_ms[-1] - 0.0) / 1000.0 max_duration = max( max_duration, min(max_duration_s, duration_s) @@ -1081,7 +1002,7 @@ def _compute_duration_and_t0_indices( except (KeyError, ValueError): continue - return max(max_duration, 1.0), t0_indices + return max(max_duration, 1.0) def _update_preprocessing_stats(self): """ diff --git a/src/tokamak_foundation_model/models/model_factory.py b/src/tokamak_foundation_model/models/model_factory.py index 27e82f7..c30f8f4 100644 --- a/src/tokamak_foundation_model/models/model_factory.py +++ b/src/tokamak_foundation_model/models/model_factory.py @@ -8,7 +8,6 @@ SpatialProfileBaselineAutoEncoder, SpectrogramBaselineAutoEncoder, SpectrogramTFAttnAutoEncoder, - SpectrogramResLSTMAutoEncoder, VideoBaselineAutoEncoder, ) From 08a2c7fb008c05f7492652bb85cac4cade6cb2fb Mon Sep 17 00:00:00 2001 From: renierts Date: Thu, 5 Mar 2026 12:31:22 -0500 Subject: [PATCH 26/30] Speed-ups in data_loader.py. --- .../data_preparation/make_processing_stats.py | 24 +- scripts/data_preparation/prepare_data.py | 14 +- scripts/slurm/make_processing_stats.sh | 2 +- scripts/slurm/prepare_data.sh | 2 +- .../data/data_loader.py | 188 ++++---- .../data/multi_file_dataset.py | 422 ++++++++++++++++++ .../data/preprocess_data.py | 385 ++++++++++++++++ 7 files changed, 916 insertions(+), 121 deletions(-) create mode 100644 src/tokamak_foundation_model/data/multi_file_dataset.py create mode 100644 src/tokamak_foundation_model/data/preprocess_data.py diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index 9bed2d6..043bc56 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -1,10 +1,11 @@ from pathlib import Path -from tokamak_foundation_model.data.data_loader import ( - TokamakH5Dataset, compute_preprocessing_stats) +from tokamak_foundation_model.data.multi_file_dataset import TokamakMultiFileDataset +from tokamak_foundation_model.data.preprocess_data import compute_preprocessing_stats + def main(): hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("*_processed.h5") + Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("20000*_processed.h5") ) all_input_signals = [ @@ -22,15 +23,16 @@ def main(): # "text", # metadata ] - datasets = [ - TokamakH5Dataset( - hdf5_path=str(f), - input_signals=all_input_signals, - target_signals=all_input_signals, - max_duration_s=10., - ) for f in hdf5_files] + dataset = TokamakMultiFileDataset( + hdf5_paths=hdf5_files, + input_signals=all_input_signals, + target_signals=all_input_signals, + lengths_cache_path="dataset_lengths.pt", + max_open_files=8, + max_duration_s=10., + ) - compute_preprocessing_stats(datasets, 'preprocessing_stats.pt') + compute_preprocessing_stats(dataset, 'preprocessing_stats_tmp.pt') if __name__ == "__main__": diff --git a/scripts/data_preparation/prepare_data.py b/scripts/data_preparation/prepare_data.py index 8b3ba34..c7ef8f7 100644 --- a/scripts/data_preparation/prepare_data.py +++ b/scripts/data_preparation/prepare_data.py @@ -591,7 +591,7 @@ def write_resampled_data( if data.size == 0 or time.size == 0: # Create minimal time axis (single point) time_out = np.array([0.0]) - data_out = np.full((num_channels, 1), np.nan, dtype='f8') + data_out = np.full((num_channels, 1), np.nan, dtype='f4') print(f" ! {group_name}: " f"No data, writing NaN array {data_out.shape}") else: @@ -604,7 +604,7 @@ def write_resampled_data( nan_channels = np.full( (missing_channels, data.shape[1]), np.nan, - dtype='f8') + dtype='f4') data_out = np.vstack([data, nan_channels]) print(f" ! {group_name}: " f"Padded {missing_channels} NaN channels") @@ -616,8 +616,8 @@ def write_resampled_data( else: data_out = data - grp.create_dataset('xdata', data=time_out, dtype='f8') - grp.create_dataset('ydata', data=data_out, dtype='f8') + grp.create_dataset('xdata', data=time_out, dtype='f4') + grp.create_dataset('ydata', data=data_out, dtype='f4') print(f" {group_name}: " f"{data_out.shape} @ {len(time_out)} samples") @@ -635,7 +635,7 @@ def write_resampled_data( # Build full data array with NaN padding data_out = np.full( - (num_channels, max_time_len), np.nan, dtype='f8') + (num_channels, max_time_len), np.nan, dtype='f4') for i, channel_data in enumerate(data): if i >= num_channels: @@ -646,8 +646,8 @@ def write_resampled_data( n_samples = min(len(channel_data), max_time_len) data_out[i, :n_samples] = channel_data[:n_samples] - grp.create_dataset('xdata', data=reference_time, dtype='f8') - grp.create_dataset('ydata', data=data_out, dtype='f8') + grp.create_dataset('xdata', data=reference_time, dtype='f4') + grp.create_dataset('ydata', data=data_out, dtype='f4') print(f" {group_name}: {data_out.shape} " f"@ {len(reference_time)} samples (from list)") diff --git a/scripts/slurm/make_processing_stats.sh b/scripts/slurm/make_processing_stats.sh index f479ea6..40a196d 100755 --- a/scripts/slurm/make_processing_stats.sh +++ b/scripts/slurm/make_processing_stats.sh @@ -5,7 +5,7 @@ #SBATCH --cpus-per-task=2 #SBATCH --nodes=1 #SBATCH --mem-per-cpu=64G -#SBATCH --time=24:00:00 +#SBATCH --time=48:00:00 #SBATCH --mail-type=all #SBATCH --mail-user=ps9551@princeton.edu diff --git a/scripts/slurm/prepare_data.sh b/scripts/slurm/prepare_data.sh index 3c9ce28..f1e2577 100755 --- a/scripts/slurm/prepare_data.sh +++ b/scripts/slurm/prepare_data.sh @@ -5,7 +5,7 @@ #SBATCH --cpus-per-task=32 # cpu-cores per task (>1 if multi-threaded tasks) #SBATCH --nodes=1 # node count #SBATCH --mem-per-cpu=16G # memory per cpu-core (4G is default) -#SBATCH --time=2:00:00 # total run time limit (HH:MM:SS) +#SBATCH --time=1:00:00 # total run time limit (HH:MM:SS) #SBATCH --mail-type=all # send email on job start, end and fault #SBATCH --mail-user=ps9551@princeton.edu diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index ca70f78..355684e 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -7,6 +7,7 @@ from typing import Optional import torch.nn.functional as F import copy +from line_profiler import profile class WelfordTensor: @@ -520,14 +521,6 @@ def __post_init__(self): self.preprocess = PreprocessConfig() -@dataclass -class ValueConfig: - """Configuration for dataloader numericals (maybe a another description)""" - - rdcc_nbytes: int # Number of bytes for the chunk cache. Adjust based on dataset size and memory constraints. - rdcc_nslots: int # Number of chunk slots in the cache. Adjust based on dataset size and access patterns. - ms_to_s: float = 1/1000 # Conversion factor from seconds to milliseconds for time calculations - class TokamakH5Dataset(Dataset): """ PyTorch Dataset for multi-modal tokamak plasma diagnostics stored in HDF5. @@ -637,10 +630,10 @@ class TokamakH5Dataset(Dataset): ``gas_flow`` 11 10 kHz no none ``gas_raw`` 11 10 kHz no none ``ich`` 1 10 kHz no none - ``mirnov`` 29 500 kHz no log - ``langmuir`` 72 500 kHz no log + ``mirnov`` 29 500 kHz yes log + ``langmuir`` 72 500 kHz yes log ``i_coil`` 18 50 kHz no none - ``bes`` 64 500 kHz no log + ``bes`` 64 500 kHz yes log ========================== ======== ========== ===== ================== **Movies** (``MOVIE_CONFIGS``) @@ -831,7 +824,7 @@ class TokamakH5Dataset(Dataset): ["mirnov"], 29, 500e3, - apply_stft=False, + apply_stft=True, preprocess=PreprocessConfig(method="log"), ), SignalConfig( @@ -839,7 +832,7 @@ class TokamakH5Dataset(Dataset): ["langmuir"], 72, 500e3, - apply_stft=False, + apply_stft=True, preprocess=PreprocessConfig(method="log"), ), SignalConfig( @@ -855,7 +848,7 @@ class TokamakH5Dataset(Dataset): ["bes"], 64, 500e3, - apply_stft=False, + apply_stft=True, preprocess=PreprocessConfig(method="log"), ), ] @@ -865,12 +858,6 @@ class TokamakH5Dataset(Dataset): MovieConfig("tangtv", ["tangtv"], 7, 50, 240, 720), ] - VALUE_CONFIG = ValueConfig( - rdcc_nbytes=1024**2 * 16, # 16 MB chunk cache - rdcc_nslots=10000, # Number of chunk slots - ms_to_s=1/1000, # Conversion factor from milliseconds to seconds - ) - def __init__( self, hdf5_path: str | Path, @@ -911,10 +898,11 @@ def __init__( self.h5_file = None try: with h5py.File(self.hdf5_path, "r") as f: - self.duration = self._compute_duration(f, max_duration_s) + duration = self._compute_duration(f) except OSError as e: print(self.hdf5_path) raise e + self.duration = min(duration, max_duration_s) # In prediction mode, reduce length to ensure extended window fits if self.prediction_mode: total_window = self.chunk_duration_s + self.prediction_horizon_s @@ -931,7 +919,6 @@ def __init__( def _compute_duration( self, f: h5py.File, - max_duration_s: float | None = None, ) -> float: """ Compute shot duration from t=0. @@ -962,17 +949,14 @@ def _compute_duration( for part in parts: curr = curr[part] - xdata_ms = curr["xdata"][:] + xdata_s = curr["xdata"][:] - if len(xdata_ms) < 2: + if len(xdata_s) < 2: continue # Duration from t=0 to end - duration_s = (xdata_ms[-1] - 0.0) / 1000.0 - max_duration = max( - max_duration, min(duration_s, max_duration_s) - ) - + duration_s = (xdata_s[-1] - 0.0) + max_duration = max(max_duration, duration_s) break except (KeyError, ValueError): @@ -992,17 +976,14 @@ def _compute_duration( if len(xdata_ms) < 2: continue - duration_s = (xdata_ms[-1] - 0.0) / 1000.0 - max_duration = max( - max_duration, min(max_duration_s, duration_s) - ) - + duration_s = (xdata_ms[-1] - 0.0) + max_duration = max(max_duration, duration_s) break except (KeyError, ValueError): continue - return max(max_duration, 1.0) + return max_duration def _update_preprocessing_stats(self): """ @@ -1030,6 +1011,7 @@ def _update_preprocessing_stats(self): if "max_val" in stats: config.preprocess.max_val = stats["max_val"] + @profile def _apply_preprocessing( self, tensor: torch.Tensor, @@ -1109,11 +1091,15 @@ def _apply_preprocessing( return (tensor - min_val) / (max_val - min_val + config.eps) elif config.method == "log_standardize": - tensor_log = torch.log10(tensor + 1) + # log10(x+1) in-place via numpy (2x faster than torch on CPU). + # tensor.numpy() is zero-copy; modifying arr updates tensor in-place. + arr = tensor.numpy() + arr += 1 + np.log10(arr, out=arr) if config.mean is None or config.std is None: print("Warning: log_standardize requested but no statistics provided") - return tensor_log + return tensor # Convert to tensor and reshape for broadcasting mean = torch.as_tensor( @@ -1125,11 +1111,13 @@ def _apply_preprocessing( mean = mean.reshape(reshape_dims) std = std.reshape(reshape_dims) - return (tensor_log - mean) / (std + config.eps) + return (tensor - mean) / (std + config.eps) elif config.method == "log": - tensor_log = torch.log10(tensor + 1) - return tensor_log + arr = tensor.numpy() + arr += 1 + np.log10(arr, out=arr) + return tensor return tensor @@ -1146,13 +1134,9 @@ def _open_hdf5(self): None """ if self.h5_file is None: - self.h5_file = h5py.File( - self.hdf5_path, - "r", - rdcc_nbytes=self.VALUE_CONFIG.rdcc_nbytes, - rdcc_nslots=self.VALUE_CONFIG.rdcc_nslots, - ) + self.h5_file = h5py.File(self.hdf5_path, "r") + @profile def _load_signal_raw( self, f: h5py.File, @@ -1177,7 +1161,7 @@ def _load_signal_raw( Returns ------- torch.Tensor - Array of shape (time_samples, channels) at native sampling rate + Array of shape (channels, time_samples) at native sampling rate """ duration_s = t_end - t_start @@ -1196,7 +1180,7 @@ def _load_signal_raw( if data_group is None: return torch.zeros( - (round(duration_s * config.target_fs), config.num_channels) + (config.num_channels, round(duration_s * config.target_fs)) ) ydata_ds = data_group["ydata"] @@ -1210,15 +1194,16 @@ def _load_signal_raw( if n_samples < 2 or xdata_end_s == xdata_start_s: return torch.zeros( - (round(duration_s * config.target_fs), config.num_channels) + (config.num_channels, round(duration_s * config.target_fs)) ) # Compute actual sampling frequency from the data actual_fs = (n_samples - 1) / (xdata_end_s - xdata_start_s) - # Step 1: Initialize output array with zeros + # Step 1: Initialize output array (C, T) — matches HDF5 storage layout, + # avoiding a transpose and keeping all copies between contiguous arrays. output = np.zeros( - (round(duration_s * actual_fs), config.num_channels), + (config.num_channels, round(duration_s * actual_fs)), dtype=np.float32 ) @@ -1232,56 +1217,55 @@ def _load_signal_raw( hdf5_start_clamped = max(0, min(hdf5_start, n_samples)) hdf5_end_clamped = max(0, min(hdf5_end, n_samples)) - # Step 3: Load data if there's any overlap + # Step 3: Load data if there's any overlap. + # Clip channels at read time so HDF5 transfers, isnan scan, and copy + # all operate on the minimum number of channels needed. if hdf5_start_clamped < hdf5_end_clamped: - data = ydata_ds[:, hdf5_start_clamped:hdf5_end_clamped].T - np.nan_to_num(data, copy=False, nan=0.0) + ch_slice = ( + config.channels_to_use + if config.channels_to_use is not None + else slice(None, config.num_channels) + ) + data = ydata_ds[ch_slice, hdf5_start_clamped:hdf5_end_clamped] # Step 4: Calculate where to insert in output array # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fs # This corresponds to output index: (that_time - t_start) * actual_fs output_start = hdf5_start_clamped - hdf5_start - output_end = output_start + data.shape[0] + output_end = output_start + data.shape[1] # Clamp to output bounds src_start = 0 - src_end = data.shape[0] + src_end = data.shape[1] if output_start < 0: src_start = -output_start output_start = 0 - if output_end > output.shape[0]: - src_end -= output_end - output.shape[0] - output_end = output.shape[0] + if output_end > output.shape[1]: + src_end -= output_end - output.shape[1] + output_end = output.shape[1] - # Insert data into output if src_start < src_end and output_start < output_end: - chunk = data[src_start:src_end] - - # Apply channel selection if specified - if config.channels_to_use is not None: - chunk = chunk[:, config.channels_to_use] + chunk = data[:, src_start:src_end] + chunk[np.isnan(chunk)] = 0 - if chunk.shape[1] == config.num_channels: - output[output_start:output_end] = chunk - elif chunk.shape[1] > config.num_channels: - output[output_start:output_end] = chunk[:, :config.num_channels] + if chunk.shape[0] == config.num_channels: + output[:, output_start:output_end] = chunk else: - output[output_start:output_end, :chunk.shape[1]] = chunk + output[:chunk.shape[0], output_start:output_end] = chunk - # Step 6: Convert to tensor and resample to target frequency - tensor = torch.from_numpy(output).float() + # Step 6: Convert to tensor and resample to target frequency. + # tensor is already (C, T), so no permute is needed around interpolate. + tensor = torch.from_numpy(output) - tensor = ( - F.interpolate( - tensor.unsqueeze(0).permute(0, 2, 1), - size=round(duration_s * config.target_fs), + T_target = round(duration_s * config.target_fs) + if tensor.shape[1] != T_target: + tensor = F.interpolate( + tensor.unsqueeze(0), + size=T_target, mode="linear", align_corners=False, - ) - .permute(0, 2, 1) - .squeeze(0) - ) + ).squeeze(0) return tensor @@ -1369,8 +1353,11 @@ def __setstate__(self, state): """Restore state after unpickling.""" self.__dict__.update(state) + @profile def _process_signal( - self, data: torch.Tensor, config: SignalConfig + self, + data: torch.Tensor, + config: SignalConfig ) -> torch.Tensor: """ Transpose, optionally compute STFT, and preprocess a raw signal. @@ -1378,7 +1365,7 @@ def _process_signal( Parameters ---------- data : torch.Tensor - Raw signal of shape ``(T, C)`` as returned by + Raw signal of shape ``(C, T)`` as returned by :meth:`_load_signal_raw`. config : SignalConfig Configuration for the signal, including ``apply_stft`` and @@ -1393,20 +1380,17 @@ def _process_signal( ``config.apply_stft`` is ``True``. - ``(C, T)`` otherwise. """ - # Step 1: Convert to torch and transpose to (channels, time) - tensor = data.T - # Step 2: Process (STFT or nothing) if config.apply_stft: - processed = self._compute_stft(tensor) + processed = self._compute_stft(data) else: - processed = tensor + processed = data # Step 3: Apply preprocessing processed = self._apply_preprocessing(processed, config.preprocess) - return processed + @profile def _load_movie_raw( self, f: h5py.File, @@ -1509,8 +1493,8 @@ def _load_movie_raw( # Step 3: Load data if there's any overlap if hdf5_start_clamped < hdf5_end_clamped: - chunk = ydata_ds[:, hdf5_start_clamped:hdf5_end_clamped, :, :] - data = np.nan_to_num(chunk, nan=0.0) + data = ydata_ds[:, hdf5_start_clamped:hdf5_end_clamped, :, :] + data[np.isnan(data)] = 0 # Step 4: Calculate where to insert in output array # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fps @@ -1534,19 +1518,20 @@ def _load_movie_raw( output[:, output_start:output_end] = data[:, src_start:src_end] # Step 5: Convert to tensor and resample to target fps and dimensions - tensor = torch.from_numpy(output).float() - - # Resample using trilinear interpolation. - # (C, T, H, W) → (1, C, T, H, W) - # → interpolate → (1, C, T', H', W') → (C, T', H', W') - tensor = ( - F.interpolate( - tensor.unsqueeze(0), # (1, C, T, H, W) - size=(round(duration_s * config.target_fps), config.height, config.width), + tensor = torch.from_numpy(output) + + # Resample using trilinear interpolation within each channel independently. + # F.interpolate treats dim-1 as channels (not interpolated across); + # the 3D kernel blends only within each channel's (T, H, W) volume. + # (C, T, H, W) → (1, C, T, H, W) → trilinear → (C, T', H', W') + target_size = (round(duration_s * config.target_fps), config.height, config.width) + if tensor.shape[1:] != torch.Size(target_size): + tensor = F.interpolate( + tensor.unsqueeze(0), + size=target_size, mode="trilinear", align_corners=False, - ).squeeze(0) # (C, T', H', W') - ) + ).squeeze(0) return tensor @@ -1577,6 +1562,7 @@ def __getitem__(self, idx: int) -> dict: else: return self._getitem_standard(idx) + @profile def _getitem_standard(self, idx: int) -> dict: """ Load and return the data chunk at *idx* in standard mode. diff --git a/src/tokamak_foundation_model/data/multi_file_dataset.py b/src/tokamak_foundation_model/data/multi_file_dataset.py new file mode 100644 index 0000000..dd6029a --- /dev/null +++ b/src/tokamak_foundation_model/data/multi_file_dataset.py @@ -0,0 +1,422 @@ +""" +Multi-file dataset for large-scale tokamak model training with HDF5 files. + +Design goals +------------ +* **Bounded file descriptors**: an LRU cache keeps at most *max_open_files* + HDF5 handles open per worker, regardless of how many files exist. +* **Sequential I/O**: :class:`TwoLevelSampler` shuffles file order each epoch + but accesses chunks sequentially *within* each file, maximising OS page-cache + hit rate and minimising seek overhead. +* **Fast startup**: file lengths (number of chunks) are written to a sidecar + ``.pt`` file on the first run and reloaded instantly on subsequent runs — + no HDF5 opens at init time after the first run. +* **No code duplication**: :class:`TokamakMultiFileDataset` subclasses + :class:`~tokamak_foundation_model.data.data_loader.TokamakH5Dataset` and + reuses all signal / movie loading methods unchanged. + +Typical usage +------------- +>>> from tokamak_foundation_model.data.multi_file_dataset import ( +... TokamakMultiFileDataset, TwoLevelSampler, make_dataloader) +>>> from torch.utils.data import DataLoader +>>> +>>> dataset = TokamakMultiFileDataset( +... hdf5_paths=sorted(Path("data/").glob("*_processed.h5")), +... input_signals=["ece", "mhr", "co2"], +... target_signals=["ece", "mhr", "co2"], +... lengths_cache_path="dataset_lengths.pt", +... max_open_files=100, +... ) +>>> loader = make_dataloader(dataset, batch_size=32, num_workers=4, shuffle=True) +>>> for batch in loader: +... ... +""" + +from __future__ import annotations + +import collections +import copy +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Optional + +import h5py +import numpy as np +import torch +from torch.utils.data import DataLoader, Sampler +from tqdm import tqdm + +from tokamak_foundation_model.data.data_loader import ( + TokamakH5Dataset, + collate_fn, + collate_fn_prediction, +) + + +class TokamakMultiFileDataset(TokamakH5Dataset): + """ + Torch Dataset spanning many HDF5 shot files with an LRU file handle cache. + + Subclasses :class:`TokamakH5Dataset` and inherits all data loading logic. + The key differences are: + + * A single dataset object covers **all** files instead of one per file. + * Open file handles are managed via a per-worker LRU cache bounded by + *max_open_files*, so file descriptor usage stays constant regardless of dataset size. + * File lengths can be persisted to a sidecar file to avoid re-scanning + HDF5 files at startup. + + Parameters + ---------- + hdf5_paths : list of str or Path + Ordered list of HDF5 shot files to include. + chunk_duration_s : float, optional + Duration of each time window in seconds. Default ``0.5``. + max_duration_s : float, optional + Maximum shot duration to consider. Default ``12.0``. + n_fft : int, optional + FFT size for STFT computation. Default ``1024``. + hop_length : int, optional + STFT hop size in samples. Default ``256``. + preprocessing_stats : dict or None, optional + Statistics dict as returned by + :func:`~tokamak_foundation_model.data.data_loader.compute_preprocessing_stats`. + prediction_mode : bool, optional + If ``True``, return ``{'inputs': …, 'targets': …}`` pairs. + prediction_horizon_s : float, optional + Target window duration in prediction mode. Default ``0.2``. + input_signals : list of str or None, optional + Modality names to include as inputs. + target_signals : list of str or None, optional + Modality names to include as targets (prediction mode only). + lengths_cache_path : str or Path or None, optional + Path to a ``.pt`` sidecar file used to cache per-file chunk counts. + On the first call the lengths are computed and written here; on + subsequent calls they are loaded instantly. ``None`` disables caching. + max_open_files : int, optional + Maximum number of HDF5 file handles kept open simultaneously **per + worker**. Default ``100``. Limits file descriptor usage; datasets are + stored contiguously so there is no active HDF5 chunk cache. + + Attributes + ---------- + hdf5_paths : list of Path + All file paths passed at construction. + _valid_indices : list of int + Indices into *hdf5_paths* for files that were successfully read. + _valid_lengths : list of int + Number of chunks in each valid file. + _cumulative_lengths : numpy.ndarray + Prefix-sum of *_valid_lengths*, used for O(log N) index mapping. + """ + + def __init__( + self, + hdf5_paths: list[str | Path], + chunk_duration_s: float = 0.5, + max_duration_s: float = 12.0, + n_fft: int = 1024, + hop_length: int = 256, + preprocessing_stats: Optional[dict] = None, + prediction_mode: bool = False, + prediction_horizon_s: float = 0.2, + input_signals: Optional[list[str]] = None, + target_signals: Optional[list[str]] = None, + lengths_cache_path: Optional[str | Path] = None, + max_open_files: int = 10_000, + ): + # Set up all instance attributes that parent methods rely on. + # We deliberately skip super().__init__() because it expects a single + # hdf5_path and opens that file — neither applies here. + self.signal_configs = copy.deepcopy(self.SIGNAL_CONFIGS) + self.movie_configs = copy.deepcopy(self.MOVIE_CONFIGS) + + self.chunk_duration_s = chunk_duration_s + self.n_fft = n_fft + self.hop_length = hop_length + self.preprocessing_stats = preprocessing_stats or {} + self.prediction_mode = prediction_mode + self.prediction_horizon_s = prediction_horizon_s + self.input_signals = input_signals or ["ece", "co2", "mhr"] + self.target_signals = target_signals or ["mse", "ts_core_density"] + self.n_freq_bins = n_fft // 2 + 1 + self.stft_window = torch.hann_window(n_fft) + # h5_file is not kept persistently; it is set in __getitem__ via the + # LRU cache so that the parent's _getitem_standard / _getitem_prediction + # methods find it on self. + self.h5_file = None + + self._update_preprocessing_stats() + + # --- multi-file state ------------------------------------------------ + self.hdf5_paths = [Path(p) for p in hdf5_paths] + self.max_open_files = max_open_files + # LRU cache: key = index into hdf5_paths, value = open h5py.File. + # OrderedDict provides O(1) move_to_end for LRU bookkeeping. + self._file_handles: collections.OrderedDict[int, h5py.File] = ( + collections.OrderedDict() + ) + + # --- lengths --------------------------------------------------------- + file_lengths = self._load_or_compute_lengths( + max_duration_s=max_duration_s, + lengths_cache_path=lengths_cache_path, + ) + + valid = [ + (i, length) for i, length in enumerate(file_lengths) if length > 0 + ] + n_skipped = len(self.hdf5_paths) - len(valid) + if n_skipped: + print( + f"Warning: {n_skipped} file(s) skipped (unreadable or empty)." + ) + + self._valid_indices: list[int] = [i for i, _ in valid] + self._valid_lengths: list[int] = [length for _, length in valid] + self._cumulative_lengths = np.concatenate( + [[0], np.cumsum(self._valid_lengths)] + ).astype(np.int64) + + # ------------------------------------------------------------------------- + # Length caching + # ------------------------------------------------------------------------- + + def _load_or_compute_lengths( + self, + max_duration_s: float, + lengths_cache_path: Optional[Path], + ) -> list[int]: + """ + Return per-file chunk counts, loading from cache when available. + + Parameters + ---------- + max_duration_s : float + Cap on shot duration used when computing chunk counts. + lengths_cache_path : Path or None + Path to the sidecar cache file. If the file exists *and* its + stored path list matches the current ``hdf5_paths``, the cached + lengths are returned directly without opening any HDF5 file. + Otherwise lengths are computed and written to this path. + + Returns + ------- + list of int + Number of chunks for each path in ``self.hdf5_paths``. + Files that could not be opened have length ``0``. + """ + paths_as_str = [str(p) for p in self.hdf5_paths] + + if lengths_cache_path is not None: + cache_path = Path(lengths_cache_path) + if cache_path.exists(): + cache = torch.load(cache_path, weights_only=False) + if cache.get("paths") == paths_as_str: + print(f"Loaded file lengths from cache: {cache_path}") + return cache["lengths"] + + lengths = [] + for path in tqdm(self.hdf5_paths, desc="Computing file lengths"): + try: + with h5py.File(path, "r") as f: + duration = min(self._compute_duration(f), max_duration_s) + if duration <= 0.0: + length = 0 + elif self.prediction_mode: + total_window = ( + self.chunk_duration_s + self.prediction_horizon_s + ) + length = max(0, int(np.floor( + (duration - total_window) / self.chunk_duration_s + ))) + else: + length = int(np.ceil(duration / self.chunk_duration_s)) + except OSError as e: + print(f"Warning: could not open {path}: {e}") + length = 0 + lengths.append(length) + + if lengths_cache_path is not None: + torch.save( + {"paths": paths_as_str, "lengths": lengths}, + lengths_cache_path + ) + print(f"Saved file lengths to cache: {lengths_cache_path}") + + return lengths + + # ------------------------------------------------------------------------- + # LRU file handle cache + # ------------------------------------------------------------------------- + + def _get_file_handle(self, file_idx: int) -> h5py.File: + """ + Return an open HDF5 handle for *file_idx*, managing an LRU cache. + + If the handle is already cached it is promoted to most-recently-used. + If the cache is full the least-recently-used handle is closed and + evicted before opening the new file. + + Parameters + ---------- + file_idx : int + Index into ``self.hdf5_paths``. + + Returns + ------- + h5py.File + Open, ready-to-read file handle. + """ + if file_idx in self._file_handles: + self._file_handles.move_to_end(file_idx) + return self._file_handles[file_idx] + + # Evict LRU entry when at capacity + if len(self._file_handles) >= self.max_open_files: + _, lru_handle = self._file_handles.popitem(last=False) + lru_handle.close() + + handle = h5py.File(self.hdf5_paths[file_idx], "r") + self._file_handles[file_idx] = handle + return handle + + # ------------------------------------------------------------------------- + # Dataset interface + # ------------------------------------------------------------------------- + + def __len__(self) -> int: + return int(self._cumulative_lengths[-1]) + + def __getitem__(self, idx: int) -> dict: + """ + Return the data chunk at global position *idx*. + + Maps *idx* to a ``(file, chunk)`` pair via binary search on the + cumulative length array, retrieves the file handle from the LRU cache, + and delegates to the parent's standard or prediction loader. + """ + # O(log N) mapping: global idx → position in valid-file list + pos = int(np.searchsorted(self._cumulative_lengths, idx + 1) - 1) + file_idx = self._valid_indices[pos] + chunk_idx = idx - int(self._cumulative_lengths[pos]) + + # Expose the handle on self so parent methods (_getitem_standard, + # _getitem_prediction, _load_signal_raw, …) can find it. + # Safe: each DataLoader worker owns its own copy of this object. + self.h5_file = self._get_file_handle(file_idx) + + if self.prediction_mode: + return self._getitem_prediction(chunk_idx) + return self._getitem_standard(chunk_idx) + + # ------------------------------------------------------------------------- + # Pickling (DataLoader worker processes) + # ------------------------------------------------------------------------- + + def __getstate__(self) -> dict: + """Close all open handles before the object is pickled to a worker.""" + state = self.__dict__.copy() + for handle in state["_file_handles"].values(): + handle.close() + state["_file_handles"] = collections.OrderedDict() + state["h5_file"] = None + return state + + def __setstate__(self, state: dict) -> None: + """ + Restore state in the worker process (file handles re-opened on demand). + """ + self.__dict__.update(state) + + +# ============================================================================= +# Two-level sampler +# ============================================================================= + +class TwoLevelSampler(Sampler): + """ + Epoch-level sampler that maximises sequential HDF5 access. + + Each epoch the list of files is shuffled (or kept in order when + ``shuffle=False``), and then the chunk indices for each file are yielded + **sequentially**. This means the DataLoader sees a different global order + each epoch while each individual file is always read front-to-back, + keeping HDF5 chunk cache utilisation high and the LRU file handle cache + effective. + + Parameters + ---------- + dataset : TokamakMultiFileDataset + The dataset to sample from. + shuffle : bool, optional + If ``True`` (default), shuffle file order at each iteration. + """ + + def __init__(self, dataset: TokamakMultiFileDataset, shuffle: bool = True): + self.dataset = dataset + self.shuffle = shuffle + + def __len__(self) -> int: + return len(self.dataset) + + def __iter__(self): + n_files = len(self.dataset._valid_lengths) + file_order = ( + torch.randperm(n_files).tolist() if self.shuffle + else list(range(n_files)) + ) + for pos in file_order: + start = int(self.dataset._cumulative_lengths[pos]) + end = int(self.dataset._cumulative_lengths[pos + 1]) + yield from range(start, end) + + +# ============================================================================= +# Convenience factory +# ============================================================================= + +def make_dataloader( + dataset: TokamakMultiFileDataset, + batch_size: int = 32, + num_workers: int = 4, + shuffle: bool = True, + pin_memory: bool = True, + prefetch_factor: int = 2, +) -> DataLoader: + """ + Build a DataLoader wired with :class:`TwoLevelSampler`. + + Parameters + ---------- + dataset : TokamakMultiFileDataset + Dataset to wrap. + batch_size : int, optional + Samples per batch. Default ``32``. + num_workers : int, optional + Number of DataLoader worker processes. Default ``4``. + shuffle : bool, optional + Whether to shuffle file order each epoch. Default ``True``. + pin_memory : bool, optional + Pin CPU tensors to accelerate CPU→GPU transfer. Default ``True``. + prefetch_factor : int, optional + Batches to prefetch per worker, overlapping I/O with GPU work. + Default ``2``. + + Returns + ------- + DataLoader + """ + sampler = TwoLevelSampler(dataset, shuffle=shuffle) + fn = collate_fn_prediction if dataset.prediction_mode else collate_fn + return DataLoader( + dataset, + batch_size=batch_size, + sampler=sampler, + num_workers=num_workers, + collate_fn=fn, + pin_memory=pin_memory, + persistent_workers=num_workers > 0, + prefetch_factor=prefetch_factor if num_workers > 0 else None, + ) diff --git a/src/tokamak_foundation_model/data/preprocess_data.py b/src/tokamak_foundation_model/data/preprocess_data.py new file mode 100644 index 0000000..9e42831 --- /dev/null +++ b/src/tokamak_foundation_model/data/preprocess_data.py @@ -0,0 +1,385 @@ +import torch +import numpy as np +from pathlib import Path +from typing import Optional +from torch.utils.data import DataLoader, SubsetRandomSampler +from .multi_file_dataset import TokamakMultiFileDataset +from .data_loader import collate_fn, collate_fn_prediction + + +class WelfordTensor: + """ + Online Welford algorithm for per-channel statistics on batched tensors. + + Accumulates running mean, variance, minimum, and maximum over an arbitrary + number of :meth:`update` calls without storing the full dataset in memory. + Statistics are computed along the channel axis (axis 1 for 3-D and 4-D + tensors) by aggregating across the batch dimension and all remaining + non-channel dimensions. Batches that contain any ``NaN`` value are + silently skipped. + + The shape of the statistics vectors depends on the input rank: + + ========= =================================== =========== + ``ndim`` Interpretation Stats shape + ========= =================================== =========== + 4 ``(B, C, F, T)`` — spectrograms / ``(C,)`` + time series + 3 ``(B, S, T)`` — profiles ``(S,)`` + ≤ 2 ``(B, T)`` or scalar — video / ``(1,)`` + fallback + ========= =================================== =========== + + Attributes + ---------- + mean : torch.Tensor or None + Running per-channel mean, shape ``(C,)``. ``None`` before the first + :meth:`update` call. + std : torch.Tensor or None + Per-channel sample standard deviation, shape ``(C,)``. Populated + only after :meth:`compute` is called. + min_val : torch.Tensor or None + Running per-channel minimum, shape ``(C,)``. ``None`` before the + first :meth:`update` call. + max_val : torch.Tensor or None + Running per-channel maximum, shape ``(C,)``. ``None`` before the + first :meth:`update` call. + n : int + Total number of scalar samples seen so far (summed over all + non-channel dimensions across all batches). + M2 : torch.Tensor or None + Running sum of squared deviations from the mean (Welford + accumulator), shape ``(C,)``. ``None`` before the first + :meth:`update` call. + initialized : bool + ``True`` once the internal buffers have been allocated on the first + :meth:`update` call. + + Notes + ----- + The parallel (batch) variant of Welford's algorithm is used to combine + each incoming batch with the accumulated state in a single pass + [1]_. All accumulation is done in ``float64`` regardless of the input + dtype to minimise floating-point cancellation errors. + + References + ---------- + .. [1] Welford, B. P. (1962). Note on a method for calculating corrected + sums of squares and products. *Technometrics*, 4(3), 419–420. + https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm + + Examples + -------- + >>> import torch + >>> tracker = WelfordTensor() + >>> for _ in range(10): + ... batch = torch.randn(32, 8, 512, 200) # (B, C, F, T) + ... tracker.update(batch) + >>> stats = tracker.compute() + >>> stats['mean'].shape + (8,) + """ + + def __init__(self): + self.mean = None + self.std = None + self.min_val = None + self.max_val = None + self.n = 0 + self.M2 = None + self.initialized = False + + def _initialize(self, value: torch.Tensor): + """ + Allocate accumulator buffers sized to match *value*. + + Called automatically by :meth:`update` on the first non-NaN batch. + Derives the number of channels from the input rank: + + * ``ndim == 4``: channel axis is 1 (spectrograms / time series). + * ``ndim == 3``: channel axis is 1 (profiles / spatial signals). + * ``ndim <= 2``: treated as single-channel (``n_channels = 1``). + + Parameters + ---------- + value : torch.Tensor + First batch tensor, used only to infer ``n_channels``. + Shape must be ``(B, C, ...)`` for 3-D or 4-D inputs. + + Returns + ------- + None + """ + # Determine number of channels based on tensor shape + # (excluding batch dim) + if value.ndim == 4: + # (batch, channels, freq_bins, time) or (batch, channels, 1, time) + n_channels = value.shape[1] + elif value.ndim == 3: + # (batch, spatial_points, time) or (batch, time, height) + # Assume spatial/channel dim is second + n_channels = value.shape[1] + elif value.ndim == 2: + # (batch, time) - single channel + n_channels = 1 + else: + # Shouldn't happen, but treat as single channel + n_channels = 1 + + self.mean = torch.zeros(n_channels, dtype=torch.float64) + self.M2 = torch.zeros(n_channels, dtype=torch.float64) + self.min_val = torch.full( + (n_channels,), float('inf'), dtype=torch.float64) + self.max_val = torch.full( + (n_channels,), float('-inf'), dtype=torch.float64) + self.initialized = True + + def update(self, value: torch.Tensor): + """ + Incorporate a new batch into the running statistics. + + Batches that contain any ``NaN`` element are silently skipped. On + the first valid call the accumulator buffers are allocated via + :meth:`_initialize`. Subsequent calls merge the incoming batch + statistics with the accumulated state using the parallel Welford + update rule. + + Parameters + ---------- + value : torch.Tensor + Batched input tensor. Supported shapes: + + * ``(B, C, F, T)`` — spectrograms or multi-channel time series. + * ``(B, C, 1, T)`` — single-frequency time series. + * ``(B, S, T)`` — spatial profiles. + * ``(B, T, H, W)`` — video frames (global statistics). + + Returns + ------- + None + """ + # Skip if contains NaN + if torch.isnan(value).any(): + return + + # Initialize on first call + if not self.initialized: + self._initialize(value) + + # Convert to float64 for numerical stability + value = value.to(dtype=torch.float64) + + # Compute per-channel statistics by flattening batch + # and all non-channel dims + if value.ndim == 4 and value.shape[1] == self.mean.shape[0]: + # (batch, channels, freq_bins, time) → flatten batch, freq, time + # (B, C, F, T) → (C, B*F*T) + n_channels = value.shape[1] + value_flat = value.permute(1, 0, 2, 3).reshape(n_channels, -1) + + # Per-channel mean, min, max + batch_mean = value_flat.mean(dim=1) + batch_min = value_flat.min(dim=1).values + batch_max = value_flat.max(dim=1).values + n_samples = value_flat.shape[1] + + # For variance, we need sum of squared deviations + batch_var = value_flat.var(dim=1, unbiased=False) + batch_M2 = batch_var * n_samples + + elif value.ndim == 3: + # (batch, spatial_points, time) → flatten batch, time + # (B, S, T) → (S, B*T) + n_channels = value.shape[1] + value_flat = value.permute(1, 0, 2).reshape(n_channels, -1) + + batch_mean = value_flat.mean(dim=1) + batch_min = value_flat.min(dim=1).values + batch_max = value_flat.max(dim=1).values + n_samples = value_flat.shape[1] + + batch_var = value_flat.var(dim=1, unbiased=False) + batch_M2 = batch_var * n_samples + + else: + # Video (batch, time, height, width) → global statistics + value_flat = value.flatten() + + batch_mean = torch.tensor([value_flat.mean()], dtype=torch.float64) + batch_min = torch.tensor([value_flat.min()], dtype=torch.float64) + batch_max = torch.tensor([value_flat.max()], dtype=torch.float64) + n_samples = value_flat.shape[0] + + batch_var = value_flat.var(unbiased=False) + batch_M2 = batch_var * n_samples + + # Parallel Welford's algorithm for combining batches + # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm + n_old = self.n + n_new = n_samples + n_total = n_old + n_new + + # Update mean + delta = batch_mean - self.mean + self.mean = (n_old * self.mean + n_new * batch_mean) / n_total + + # Update M2 (sum of squared deviations) + # M2_total = M2_old + M2_new + delta^2 * n_old * n_new / n_total + self.M2 = self.M2 + batch_M2 + delta * delta * n_old * n_new / n_total + + self.n = n_total + + # Update min/max + self.min_val = torch.minimum(self.min_val, batch_min) + self.max_val = torch.maximum(self.max_val, batch_max) + + def _compute_std(self): + """ + Derive sample standard deviation from the Welford M2 accumulator. + + Uses Bessel's correction (``n - 1``) when more than one sample has + been seen; falls back to zeros when ``n <= 1`` to avoid division by + zero. The result is written to :attr:`std` in-place. + + Returns + ------- + None + """ + if self.n > 1: + self.std = torch.sqrt(self.M2 / (self.n - 1)) + else: + self.std = torch.zeros_like(self.mean) + + def compute(self): + """ + Finalise and return all accumulated statistics as NumPy arrays. + + Calls :meth:`_compute_std` internally to derive the standard + deviation from the Welford M2 accumulator before returning. + Returns ``None`` if :meth:`update` was never called. + + Returns + ------- + dict or None + ``None`` if no data was ever seen. Otherwise a dictionary + with the following keys, each mapping to a + ``numpy.ndarray`` of shape ``(C,)``: + + ``'mean'`` + Per-channel arithmetic mean. + ``'std'`` + Per-channel sample standard deviation (Bessel-corrected). + ``'min_val'`` + Per-channel minimum value seen across all batches. + ``'max_val'`` + Per-channel maximum value seen across all batches. + """ + if not self.initialized: + return None + + self._compute_std() + + return { + "mean": self.mean.numpy(), + "std": self.std.numpy(), + "min_val": self.min_val.numpy(), + "max_val": self.max_val.numpy(), + } + + +def compute_preprocessing_stats( + dataset: TokamakMultiFileDataset, + output_path: str | Path = "preprocessing_stats.pt", + batch_size: int = 1, + num_workers: int = 0, + max_chunks: Optional[int] = 10_000, +) -> dict[str, dict[str, np.ndarray]]: + """ + Compute per-modality preprocessing statistics over a dataset. + + Accumulates running statistics with :class:`WelfordTensor` and saves the + result to *output_path* via :func:`torch.save`. Only modalities that + appear in the loaded batches are included in the output. + + Parameters + ---------- + dataset : TokamakMultiFileDataset + Dataset to compute statistics over. + output_path : str or Path, optional + Filesystem path for the saved ``.pt`` statistics file. + Default is ``"preprocessing_stats.pt"``. + batch_size : int, optional + Batch size for the internal DataLoader. Default is ``1``. + num_workers : int, optional + Number of DataLoader worker processes. Default is ``0`` (main + process only). Workers add IPC overhead that outweighs any benefit + for this CPU-only, I/O-bound task. + max_chunks : int or None, optional + Maximum number of chunks to sample from the dataset. A random + subset of this size is drawn without replacement. ``None`` means + use the full dataset. Default is ``10_000``, which gives accurate + statistics in ~1-2 hours instead of hundreds of hours. + + Returns + ------- + dict[str, dict[str, numpy.ndarray]] + Nested dictionary ``{modality_name: stats}``, where *stats* is the + dictionary returned by :meth:`WelfordTensor.compute`: + + ``'mean'`` + Per-channel arithmetic mean, shape ``(C,)``. + ``'std'`` + Per-channel sample standard deviation, shape ``(C,)``. + ``'min_val'`` + Per-channel minimum, shape ``(C,)``. + ``'max_val'`` + Per-channel maximum, shape ``(C,)``. + """ + from tqdm import tqdm + + # Use instance-level configs (deep copies that may have been modified). + signal_configs = dataset.signal_configs + movie_configs = dataset.movie_configs + + welford_stats = { + cfg.name: WelfordTensor() + for cfg in signal_configs + movie_configs} + + n_total = len(dataset) + if max_chunks is not None and max_chunks < n_total: + indices = torch.randperm(n_total)[:max_chunks].tolist() + print(f"Subsampling {max_chunks:,} / {n_total:,} chunks for statistics.") + else: + indices = list(range(n_total)) + + collate = collate_fn_prediction if dataset.prediction_mode else collate_fn + dataloader = DataLoader( + dataset, + batch_size=batch_size, + sampler=SubsetRandomSampler(indices), + num_workers=num_workers, + collate_fn=collate, + pin_memory=False, + ) + + for batch in tqdm(dataloader, total=len(indices) // batch_size): + for modality_name, tensor in batch.items(): + if modality_name not in welford_stats: + continue + # Movies arrive as (B, C, T, H, W); flatten spatial/temporal dims + # to (B, C, T*H*W) so WelfordTensor computes per-channel stats. + if tensor.ndim == 5: + B, C, T, H, W = tensor.shape + tensor = tensor.reshape(B, C, T * H * W) + welford_stats[modality_name].update(tensor) + + # Only include trackers that received data + final_stats = { + modality: tracker.compute() + for modality, tracker in welford_stats.items() + if tracker.initialized + } + torch.save(final_stats, output_path) + + print(f"Saved statistics to {output_path}") + return final_stats From 6a55406418e54fac37e8248fb76a36618b0b1a0a Mon Sep 17 00:00:00 2001 From: renierts Date: Mon, 9 Mar 2026 16:14:55 -0400 Subject: [PATCH 27/30] Speed-ups in the dataloader. Bugfixes in the trainer. Cosmetic changes in tracking.py --- pixi.lock | 808 +++++++++++++++++- pyproject.toml | 4 + scripts/data_fetching_omega/read_mds.sh | 226 ++--- .../submit_read_mds_batches.sh | 14 +- .../data_preparation/make_processing_stats.py | 4 +- scripts/data_preparation/prepare_data.py | 3 + scripts/slurm/prepare_data.sh | 2 +- scripts/slurm/train_filterscopes.sh | 26 + .../fast_time_series_reconstruction.py | 100 ++- .../data/config/config.yaml | 2 +- .../data/data_loader.py | 482 ++--------- .../data/multi_file_dataset.py | 4 + .../data/preprocess_data.py | 4 +- .../models/model_factory.py | 3 +- .../trainer/trainer.py | 121 +-- src/tokamak_foundation_model/utils/drawing.py | 2 +- .../utils/tracking.py | 31 +- 17 files changed, 1209 insertions(+), 627 deletions(-) create mode 100644 scripts/slurm/train_filterscopes.sh diff --git a/pixi.lock b/pixi.lock index c7e0438..e595906 100644 --- a/pixi.lock +++ b/pixi.lock @@ -30,6 +30,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/line_profiler-5.0.2-py311h724c32c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.1-h35e630c_1.conda @@ -43,7 +44,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/88/3f/e1b801e3b56a356f799f604adaaaaffbe2a4fdb902e035c4cc11bd90bc6f/blosc2-4.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -62,6 +65,9 @@ environments: - pypi: https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/61/1ca198af22f7dd22c17ab86e9024ed3c06299cfdb08170640e9996d501a0/fonttools-4.61.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/61/8ac32517c1e856677282c34f2e7812d6c328fa02b8f4067ab80e77fdc9c9/grpcio-1.78.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8b/23/4ab1108e87851ccc69694b03b817d92e142966a6c4abd99e17db77f2c066/h5py-3.15.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/9a/92/cf3ab0b652b082e66876d08da57fcc6fa2f0e6c70dfbbafbd470bb73eb47/hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -79,6 +85,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/66/e1/e533435c0be77c3f64040d68d7a657771194a63c279f55573188161e81ca/kiwisolver-1.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8f/a0/7024215e95d456de5883e6732e708d8187d9753a21d32f8ddb3befc0c445/matplotlib-3.10.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl @@ -112,10 +120,13 @@ environments: - pypi: https://files.pythonhosted.org/packages/a2/c8/46dfeac5825e600579157eea177be43e2f7ff4a99da9d0d0a49533509ac5/pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/9b/53/a9443aa3ca9ba8724fdfa02dd1887c1bcd8e89556b715cfbacca6b63dbec/protobuf-6.33.5-cp39-abi3-manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl @@ -125,14 +136,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ef/df/df1457c4df3826e908879fe3d76bc5b6e60aae45f4ee42539512438cfd5d/scipy-1.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/88/d5/71665919aa2a5a3d2a20eeef3c71dc7c2ebbd9f26d114a7808514aba24d6/tables-3.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://download.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl - pypi: https://download.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/50/d4/e51d52047e7eb9a582da59f32125d17c0482d065afd5d3bc435ff2120dc5/tornado-6.5.4-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl @@ -141,8 +158,11 @@ environments: - pypi: https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/91/ec9465d014cfd199c5b2083d271d31b3c2aedeae66f3d8a0712f7f54bdf3/wandb-0.25.0-py3-none-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ osx-arm64: @@ -151,11 +171,13 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.1.4-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hydra-core-1.3.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.2-h38cb7af_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-22.1.0-h55c6f16_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.7.3-haf25636_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.5.2-hcf2aa1b_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.8.2-h8088a28_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.51.2-h1ae2325_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.3.1-h8359307_2.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/line_profiler-5.0.2-py311h7d85929_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-h5e97a16_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.6.1-hd24854e_1.conda @@ -168,7 +190,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl @@ -186,6 +210,9 @@ environments: - pypi: https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/12/bf9f4eaa2fad039356cc627587e30ed008c03f1cebd3034376b5ee8d1d44/fonttools-4.61.1-cp311-cp311-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c5/b1/96920bf2ee61df85a9503cb6f733fe711c0ff321a5a697d791b075673281/grpcio-1.78.0-cp311-cp311-macosx_11_0_universal2.whl - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/b0/1c628e26a0b95858f54aba17e1599e7f6cd241727596cc2580b72cb0a9bf/h5py-3.15.1-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/7f/8c/c5becfa53234299bc2210ba314eaaae36c2875e0045809b82e40a9544f0c/hf_xet-1.2.0-cp37-abi3-macosx_11_0_arm64.whl @@ -203,6 +230,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/31/a2/a12a503ac1fd4943c50f9822678e8015a790a13b5490354c68afb8489814/kiwisolver-1.4.9-cp311-cp311-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/fd/14/baad3222f424b19ce6ad243c71de1ad9ec6b2e4eb1e458a48fdc6d120401/matplotlib-3.10.8-cp311-cp311-macosx_11_0_arm64.whl @@ -221,10 +250,13 @@ environments: - pypi: https://files.pythonhosted.org/packages/78/93/a29e9bc02d1cf557a834da780ceccd54e02421627200696fcf805ebdc3fb/pillow-12.1.1-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a2/6b/e48dfc1191bc5b52950246275bf4089773e91cb5ba3592621723cdddca62/protobuf-6.33.5-cp39-abi3-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl @@ -234,14 +266,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e8/00/374c0c068e30cd31f1e1b46b4b5738168ec79e7689ca82ee93ddfea05109/safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/5e/5f/a6b38f79a07d74989224d5f11b55267714707582908a5f1ae854cf9a9b84/scipy-1.17.0-cp311-cp311-macosx_12_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d9/d0/accd41382fa9da45bf816c56f85bda64223a3b8d0006d3496b67e0781a6e/tables-3.10.2-cp311-cp311-macosx_11_0_arm64.whl + - pypi: https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://download.pytorch.org/whl/cpu/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl - pypi: https://download.pytorch.org/whl/cpu/torchvision-0.25.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/ab/a9/e94a9d5224107d7ce3cc1fab8d5dc97f5ea351ccc6322ee4fb661da94e35/tornado-6.5.4-cp39-abi3-macosx_10_9_universal2.whl - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl @@ -249,8 +287,11 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/66/57042d4b0f1ede8046d7ae6409bf3640df996e9cbc3fe20467aa29badc54/transformers-5.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c1/7d/0c131db3ec9deaabbd32263d90863cbfbe07659527e11c35a5c738cecdc5/wandb-0.25.0-py3-none-macosx_12_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ win-64: @@ -263,6 +304,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/liblzma-5.8.2-hfd05255_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libsqlite-3.51.2-hf5d6505_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libzlib-1.3.1-h2466b09_2.conda + - conda: https://conda.anaconda.org/conda-forge/win-64/line_profiler-5.0.2-py311h275cad7_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/omegaconf-2.3.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/openssl-3.6.1-hf411b9b_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda @@ -277,7 +319,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/vc14_runtime-14.44.35208-h818238b_34.conda - conda: https://conda.anaconda.org/conda-forge/win-64/vcomp14-14.44.35208-h818238b_34.conda - conda: https://conda.anaconda.org/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c1/01/6ff32c4e6e13069f226cddf14abc0f075b8699e345e2d411b6874135b421/blosc2-4.0.0-cp311-cp311-win_amd64.whl @@ -295,6 +339,9 @@ environments: - pypi: https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/07/ad/37dd1ae5fa6e01612a1fbb954f0927681f282925a86e86198ccd7b15d515/fonttools-4.61.1-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/08/62/f22c98c5265dfad327251fa2f840b591b1df5f5e15d88b19c18c86965b27/grpcio-1.78.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/23/95/499b4e56452ef8b6c95a271af0dde08dac4ddb70515a75f346d4f400579b/h5py-3.15.1-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl @@ -312,6 +359,8 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ab/b5/36c712098e6191d1b4e349304ef73a8d06aed77e56ceaac8c0a306c7bda1/jupyterlab_widgets-3.0.16-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3b/c6/f8df8509fd1eee6c622febe54384a96cfaf4d43bf2ccec7a0cc17e4715c9/kiwisolver-1.4.9-cp311-cp311-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/6f/d3/a4bbc01c237ab710a1f22b4da72f4ff6d77eb4c7735ea9811a94ae239067/matplotlib-3.10.8-cp311-cp311-win_amd64.whl @@ -329,9 +378,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/55/75/bb9bc917d10e9ee13dee8607eb9ab963b7cf8be607c46e7862c748aa2af7/protobuf-6.33.5-cp310-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl @@ -341,14 +393,20 @@ environments: - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/5d/e6/ec8471c8072382cb91233ba7267fd931219753bb43814cbc71757bfd4dab/safetensors-0.7.0-cp38-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/52/c8/08629657ac6c0da198487ce8cd3de78e02cfde42b7f34117d56a3fe249dc/scipy-1.17.0-cp311-cp311-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/46/96/b5023c1f7b9d560cac3e2c0daceebaeb88dd24c70c75db2d291abfa563e5/tables-3.10.2-cp311-cp311-win_amd64.whl + - pypi: https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl - pypi: https://download.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl - pypi: https://download.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/d6/6d/c69be695a0a64fd37a97db12355a035a6d90f79067a3cf936ec2b1dc38cd/tornado-6.5.4-cp39-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl @@ -356,9 +414,12 @@ environments: - pypi: https://files.pythonhosted.org/packages/b7/66/57042d4b0f1ede8046d7ae6409bf3640df996e9cbc3fe20467aa29badc54/transformers-5.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/25/97/460f6cb738aaa39b4eb2e6b4c630b2ae4321cdd70a79d5955ea75a878981/wandb-0.25.0-py3-none-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl - pypi: ./ fdp: @@ -522,6 +583,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxml2-2.13.9-h04c0eec_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/line_profiler-5.0.2-py311h724c32c_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/markupsafe-3.0.3-py311h3778330_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/matplotlib-inline-0.2.1-pyhd8ed1ab_0.conda @@ -629,7 +691,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.25.0-py311haee01d2_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda + - pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/88/3f/e1b801e3b56a356f799f604adaaaaffbe2a4fdb902e035c4cc11bd90bc6f/blosc2-4.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/5f/4b/6157f24ca425b89fe2eb7e7be642375711ab671135be21e6faa100f7448c/contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/45/e7/b47792cc2d01c7e1d37c32402182524774dadd2d26339bd224e0e913832e/cuda_bindings-12.9.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl @@ -637,10 +701,15 @@ environments: - pypi: https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/61/1ca198af22f7dd22c17ab86e9024ed3c06299cfdb08170640e9996d501a0/fonttools-4.61.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e5/61/8ac32517c1e856677282c34f2e7812d6c328fa02b8f4067ab80e77fdc9c9/grpcio-1.78.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/8b/23/4ab1108e87851ccc69694b03b817d92e142966a6c4abd99e17db77f2c066/h5py-3.15.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/9a/92/cf3ab0b652b082e66876d08da57fcc6fa2f0e6c70dfbbafbd470bb73eb47/hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d5/ae/2f6d96b4e6c5478d87d606a1934b5d436c4a2bce6bb7c6fdece891c128e3/huggingface_hub-1.4.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/66/e1/e533435c0be77c3f64040d68d7a657771194a63c279f55573188161e81ca/kiwisolver-1.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8f/a0/7024215e95d456de5883e6732e708d8187d9753a21d32f8ddb3befc0c445/matplotlib-3.10.8-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl @@ -665,22 +734,32 @@ environments: - pypi: https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a2/c8/46dfeac5825e600579157eea177be43e2f7ff4a99da9d0d0a49533509ac5/pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a4/3c/87ca0a02736d16b6262921425e84b48984e77d8e4e572c9072ce96e66c30/regex-2026.1.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a0/60/429e9b1cb3fc651937727befe258ea24122d9663e4d5709a48c9cbfceecb/safetensors-0.7.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/88/d5/71665919aa2a5a3d2a20eeef3c71dc7c2ebbd9f26d114a7808514aba24d6/tables-3.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://download.pytorch.org/whl/cu128/torch-2.10.0%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/72/25/973bd6128381951b23cdcd8a9870c6dcfc5606cb864df8eabd82e529f9c1/torchinfo-1.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl - pypi: https://download.pytorch.org/whl/cu128/torchvision-0.25.0%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/66/57042d4b0f1ede8046d7ae6409bf3640df996e9cbc3fe20467aa29badc54/transformers-5.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/4b/e7/61b0dd194be67021ff7c6c87b66511d7691b9b241b2a67a2a5e3842e531b/typer-0.22.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c0/fc/a2fe203a85b998556dfaca0704d3a76a1e39b3301a0ca7013d68b054d84c/typer_slim-0.22.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/de/91/ec9465d014cfd199c5b2083d271d31b3c2aedeae66f3d8a0712f7f54bdf3/wandb-0.25.0-py3-none-manylinux_2_28_x86_64.whl + - pypi: https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl - pypi: ./ packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -704,6 +783,11 @@ packages: purls: [] size: 23621 timestamp: 1650670423406 +- pypi: https://files.pythonhosted.org/packages/18/a6/907a406bb7d359e6a63f99c313846d9eec4f7e6f7437809e03aa00fa3074/absl_py-2.4.0-py3-none-any.whl + name: absl-py + version: 2.4.0 + sha256: 88476fd881ca8aab94ffa78b7b6c632a782ab3ba1cd19c9bd423abc4fb4cd28d + requires_python: '>=3.10' - conda: https://conda.anaconda.org/conda-forge/noarch/aiohappyeyeballs-2.6.1-pyhd8ed1ab_0.conda sha256: 7842ddc678e77868ba7b92a726b437575b23aaec293bca0d40826f1026d90e27 md5: 18fd895e0e775622906cdabfc3cf0fb4 @@ -754,6 +838,13 @@ packages: version: 0.0.4 sha256: 571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 requires_python: '>=3.8' +- pypi: https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl + name: annotated-types + version: 0.7.0 + sha256: 1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53 + requires_dist: + - typing-extensions>=4.0.0 ; python_full_version < '3.9' + requires_python: '>=3.8' - conda: https://conda.anaconda.org/conda-forge/noarch/antlr-python-runtime-4.9.3-pyhd8ed1ab_1.tar.bz2 sha256: b91f8ab4ac2b48972fbee1fc8e092cc452fdf59156e4ff2322c94bbf73650f94 md5: c88eaec8de9ae1fa161205aa18e7a5b1 @@ -1769,10 +1860,11 @@ packages: - pypi: ./ name: faith version: 26.1.dev0 - sha256: 947201fad263cc81e9052dd4afa8eef157340bf2839eae66cbb7558ce7d0d073 + sha256: 8da1a100c63a498d6f2ffab9e15845ab297cb641bb16309badf1946cc1264b5c requires_dist: - einops>=0.8.2,<0.9 - h5py>=3.15.1,<4 + - hydra-core - ipykernel>=7.2.0,<8 - ipywidgets>=8.1.8,<9 - matplotlib>=3.10.8,<4 @@ -1780,10 +1872,13 @@ packages: - pandas>=3.0.0,<4 - scipy - tables>=3.10.2,<4 + - tensorboard - torch - torchinfo>=1.8.0,<2 + - torchmetrics>=1.6.0,<2 - torchvision - transformers>=5.1.0,<6 + - wandb requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl name: filelock @@ -2077,6 +2172,35 @@ packages: purls: [] size: 119654 timestamp: 1726600001928 +- pypi: https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl + name: gitdb + version: 4.0.12 + sha256: 67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf + requires_dist: + - smmap>=3.0.1,<6 + requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/6a/09/e21df6aef1e1ffc0c816f0522ddc3f6dcded766c3261813131c78a704470/gitpython-3.1.46-py3-none-any.whl + name: gitpython + version: 3.1.46 + sha256: 79812ed143d9d25b6d176a10bb511de0f9c67b1fa641d82097b0ab90398a2058 + requires_dist: + - gitdb>=4.0.1,<5 + - typing-extensions>=3.10.0.2 ; python_full_version < '3.10' + - coverage[toml] ; extra == 'test' + - ddt>=1.1.1,!=1.4.3 ; extra == 'test' + - mock ; python_full_version < '3.8' and extra == 'test' + - mypy==1.18.2 ; python_full_version >= '3.9' and extra == 'test' + - pre-commit ; extra == 'test' + - pytest>=7.3.1 ; extra == 'test' + - pytest-cov ; extra == 'test' + - pytest-instafail ; extra == 'test' + - pytest-mock ; extra == 'test' + - pytest-sugar ; extra == 'test' + - typing-extensions ; python_full_version < '3.11' and extra == 'test' + - sphinx>=7.1.2,<7.2 ; extra == 'doc' + - sphinx-rtd-theme ; extra == 'doc' + - sphinx-autodoc-typehints ; extra == 'doc' + requires_python: '>=3.7' - conda: https://conda.anaconda.org/conda-forge/linux-64/glog-0.7.1-hbabe93e_0.conda sha256: dc824dc1d0aa358e28da2ecbbb9f03d932d976c8dca11214aa1dcdfcbd054ba2 md5: ff862eebdfeb2fd048ae9dc92510baca @@ -2104,6 +2228,30 @@ packages: - pkg:pypi/google-crc32c?source=hash-mapping size: 25242 timestamp: 1768549195622 +- pypi: https://files.pythonhosted.org/packages/08/62/f22c98c5265dfad327251fa2f840b591b1df5f5e15d88b19c18c86965b27/grpcio-1.78.0-cp311-cp311-win_amd64.whl + name: grpcio + version: 1.78.0 + sha256: 1afa62af6e23f88629f2b29ec9e52ec7c65a7176c1e0a83292b93c76ca882558 + requires_dist: + - typing-extensions~=4.12 + - grpcio-tools>=1.78.0 ; extra == 'protobuf' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c5/b1/96920bf2ee61df85a9503cb6f733fe711c0ff321a5a697d791b075673281/grpcio-1.78.0-cp311-cp311-macosx_11_0_universal2.whl + name: grpcio + version: 1.78.0 + sha256: 9dca934f24c732750389ce49d638069c3892ad065df86cb465b3fa3012b70c9e + requires_dist: + - typing-extensions~=4.12 + - grpcio-tools>=1.78.0 ; extra == 'protobuf' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/e5/61/8ac32517c1e856677282c34f2e7812d6c328fa02b8f4067ab80e77fdc9c9/grpcio-1.78.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl + name: grpcio + version: 1.78.0 + sha256: 85f93781028ec63f383f6bc90db785a016319c561cc11151fbb7b34e0d012303 + requires_dist: + - typing-extensions~=4.12 + - grpcio-tools>=1.78.0 ; extra == 'protobuf' + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl name: h11 version: 0.16.0 @@ -3385,6 +3533,16 @@ packages: purls: [] size: 462942 timestamp: 1767821743793 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-22.1.0-h55c6f16_1.conda + sha256: ce1049fa6fda9cf08ff1c50fb39573b5b0ea6958375d8ea7ccd8456ab81a0bcb + md5: e9c56daea841013e7774b5cd46f41564 + depends: + - __osx >=11.0 + license: Apache-2.0 WITH LLVM-exception + license_family: Apache + purls: [] + size: 568910 + timestamp: 1772001095642 - conda: https://conda.anaconda.org/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda sha256: d789471216e7aba3c184cd054ed61ce3f6dac6f87a50ec69291b9297f8c18724 md5: c277e0a4d549b03ac1e9d6cbbe3d017b @@ -3982,6 +4140,76 @@ packages: purls: [] size: 55476 timestamp: 1727963768015 +- pypi: https://files.pythonhosted.org/packages/25/f4/ead6e0e37209b07c9baa3e984ccdb0348ca370b77cea3aaea8ddbb097e00/lightning_utilities-0.15.3-py3-none-any.whl + name: lightning-utilities + version: 0.15.3 + sha256: 6c55f1bee70084a1cbeaa41ada96e4b3a0fea5909e844dd335bd80f5a73c5f91 + requires_dist: + - packaging>=22 + - typing-extensions + - mypy>=1.0.0 ; extra == 'typing' + - types-setuptools ; extra == 'typing' + - requests>=2.0.0 ; extra == 'docs' + - jsonargparse[signatures]>=4.38.0 ; extra == 'cli' + - tomlkit ; extra == 'cli' + requires_python: '>=3.10' +- conda: https://conda.anaconda.org/conda-forge/linux-64/line_profiler-5.0.2-py311h724c32c_0.conda + sha256: d62439e2a2f8135914832d10e3a0ecf9ded866b23fb505bad19483e36906ddf1 + md5: 67e7266f73026642f384aa169a5391c1 + depends: + - python + - typing_extensions + - libstdcxx >=14 + - libgcc >=14 + - __glibc >=2.17,<3.0.a0 + - python_abi 3.11.* *_cp311 + constrains: + - ipython >=8.14.0 + - rich >=12.3.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/line-profiler?source=hash-mapping + size: 529685 + timestamp: 1771974558950 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/line_profiler-5.0.2-py311h7d85929_0.conda + sha256: 115ec27ec36899f378f0a16cb55ec4417e4d3bf0fdb5cd42a67afb9c820a8e97 + md5: 32e9d84be6cb4b3cde1f3044ba0b106e + depends: + - python + - typing_extensions + - python 3.11.* *_cpython + - libcxx >=19 + - __osx >=11.0 + - python_abi 3.11.* *_cp311 + constrains: + - ipython >=8.14.0 + - rich >=12.3.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/line-profiler?source=hash-mapping + size: 506377 + timestamp: 1771974728643 +- conda: https://conda.anaconda.org/conda-forge/win-64/line_profiler-5.0.2-py311h275cad7_0.conda + sha256: 3eebabc4d4b53ff1425de7b53172e8ef63a927a6b63a15fb40c13f244cba7971 + md5: 37723cf3808e0f858f4240a4f0c67c39 + depends: + - python + - typing_extensions + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + - python_abi 3.11.* *_cp311 + constrains: + - ipython >=8.14.0 + - rich >=12.3.0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/line-profiler?source=hash-mapping + size: 535877 + timestamp: 1771974573512 - conda: https://conda.anaconda.org/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda sha256: 47326f811392a5fd3055f0f773036c392d26fdb32e4d8e7a8197eed951489346 md5: 9de5350a85c4a20c685259b889aa6393 @@ -3994,6 +4222,21 @@ packages: purls: [] size: 167055 timestamp: 1733741040117 +- pypi: https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl + name: markdown + version: 3.10.2 + sha256: e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36 + requires_dist: + - coverage ; extra == 'testing' + - pyyaml ; extra == 'testing' + - mkdocs>=1.6 ; extra == 'docs' + - mkdocs-nature>=0.6 ; extra == 'docs' + - mdx-gh-links>=0.2 ; extra == 'docs' + - mkdocstrings[python]>=0.28.3 ; extra == 'docs' + - mkdocs-gen-files ; extra == 'docs' + - mkdocs-section-index ; extra == 'docs' + - mkdocs-literate-nav ; extra == 'docs' + requires_python: '>=3.10' - pypi: https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl name: markdown-it-py version: 4.0.0 @@ -5282,6 +5525,21 @@ packages: - pkg:pypi/propcache?source=hash-mapping size: 54558 timestamp: 1744525097548 +- pypi: https://files.pythonhosted.org/packages/55/75/bb9bc917d10e9ee13dee8607eb9ab963b7cf8be607c46e7862c748aa2af7/protobuf-6.33.5-cp310-abi3-win_amd64.whl + name: protobuf + version: 6.33.5 + sha256: 3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/9b/53/a9443aa3ca9ba8724fdfa02dd1887c1bcd8e89556b715cfbacca6b63dbec/protobuf-6.33.5-cp39-abi3-manylinux2014_x86_64.whl + name: protobuf + version: 6.33.5 + sha256: cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/a2/6b/e48dfc1191bc5b52950246275bf4089773e91cb5ba3592621723cdddca62/protobuf-6.33.5-cp39-abi3-macosx_10_9_universal2.whl + name: protobuf + version: 6.33.5 + sha256: a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5 + requires_python: '>=3.9' - conda: https://conda.anaconda.org/conda-forge/linux-64/protobuf-6.31.1-py311h425ed32_2.conda sha256: f5216cb89239542d39b9dfc9a757157f8c779e88a769c165e275da035b38cd02 md5: 28ef5e67a2544510913d04a4a6dd9e12 @@ -5555,6 +5813,39 @@ packages: - pkg:pypi/pycparser?source=hash-mapping size: 110100 timestamp: 1733195786147 +- pypi: https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl + name: pydantic + version: 2.12.5 + sha256: e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d + requires_dist: + - annotated-types>=0.6.0 + - pydantic-core==2.41.5 + - typing-extensions>=4.14.1 + - typing-inspection>=0.4.2 + - email-validator>=2.0.0 ; extra == 'email' + - tzdata ; python_full_version >= '3.9' and sys_platform == 'win32' and extra == 'timezone' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl + name: pydantic-core + version: 2.41.5 + sha256: 76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl + name: pydantic-core + version: 2.41.5 + sha256: 7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: pydantic-core + version: 2.41.5 + sha256: f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b + requires_dist: + - typing-extensions>=4.14.1 + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl name: pygments version: 2.19.2 @@ -6363,6 +6654,123 @@ packages: - pkg:pypi/send2trash?source=hash-mapping size: 23960 timestamp: 1768402421616 +- pypi: https://files.pythonhosted.org/packages/53/39/be412cc86bc6247b8f69e9383d7950711bd86f8d0a4a4b0fe8fad685bc21/sentry_sdk-2.54.0-py2.py3-none-any.whl + name: sentry-sdk + version: 2.54.0 + sha256: fd74e0e281dcda63afff095d23ebcd6e97006102cdc8e78a29f19ecdf796a0de + requires_dist: + - urllib3>=1.26.11 + - certifi + - aiohttp>=3.5 ; extra == 'aiohttp' + - anthropic>=0.16 ; extra == 'anthropic' + - arq>=0.23 ; extra == 'arq' + - asyncpg>=0.23 ; extra == 'asyncpg' + - apache-beam>=2.12 ; extra == 'beam' + - bottle>=0.12.13 ; extra == 'bottle' + - celery>=3 ; extra == 'celery' + - celery-redbeat>=2 ; extra == 'celery-redbeat' + - chalice>=1.16.0 ; extra == 'chalice' + - clickhouse-driver>=0.2.0 ; extra == 'clickhouse-driver' + - django>=1.8 ; extra == 'django' + - falcon>=1.4 ; extra == 'falcon' + - fastapi>=0.79.0 ; extra == 'fastapi' + - flask>=0.11 ; extra == 'flask' + - blinker>=1.1 ; extra == 'flask' + - markupsafe ; extra == 'flask' + - grpcio>=1.21.1 ; extra == 'grpcio' + - protobuf>=3.8.0 ; extra == 'grpcio' + - httpcore[http2]==1.* ; extra == 'http2' + - httpx>=0.16.0 ; extra == 'httpx' + - huey>=2 ; extra == 'huey' + - huggingface-hub>=0.22 ; extra == 'huggingface-hub' + - langchain>=0.0.210 ; extra == 'langchain' + - langgraph>=0.6.6 ; extra == 'langgraph' + - launchdarkly-server-sdk>=9.8.0 ; extra == 'launchdarkly' + - litellm>=1.77.5 ; extra == 'litellm' + - litestar>=2.0.0 ; extra == 'litestar' + - loguru>=0.5 ; extra == 'loguru' + - mcp>=1.15.0 ; extra == 'mcp' + - openai>=1.0.0 ; extra == 'openai' + - tiktoken>=0.3.0 ; extra == 'openai' + - openfeature-sdk>=0.7.1 ; extra == 'openfeature' + - opentelemetry-distro>=0.35b0 ; extra == 'opentelemetry' + - opentelemetry-distro ; extra == 'opentelemetry-experimental' + - opentelemetry-distro[otlp]>=0.35b0 ; extra == 'opentelemetry-otlp' + - pure-eval ; extra == 'pure-eval' + - executing ; extra == 'pure-eval' + - asttokens ; extra == 'pure-eval' + - pydantic-ai>=1.0.0 ; extra == 'pydantic-ai' + - pymongo>=3.1 ; extra == 'pymongo' + - pyspark>=2.4.4 ; extra == 'pyspark' + - quart>=0.16.1 ; extra == 'quart' + - blinker>=1.1 ; extra == 'quart' + - rq>=0.6 ; extra == 'rq' + - sanic>=0.8 ; extra == 'sanic' + - sqlalchemy>=1.2 ; extra == 'sqlalchemy' + - starlette>=0.19.1 ; extra == 'starlette' + - starlite>=1.48 ; extra == 'starlite' + - statsig>=0.55.3 ; extra == 'statsig' + - tornado>=6 ; extra == 'tornado' + - unleashclient>=6.0.1 ; extra == 'unleash' + - google-genai>=1.29.0 ; extra == 'google-genai' + requires_python: '>=3.6' +- pypi: https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl + name: setuptools + version: 82.0.0 + sha256: 70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0 + requires_dist: + - pytest>=6,!=8.1.* ; extra == 'test' + - virtualenv>=13.0.0 ; extra == 'test' + - wheel>=0.44.0 ; extra == 'test' + - pip>=19.1 ; extra == 'test' + - packaging>=24.2 ; extra == 'test' + - jaraco-envs>=2.2 ; extra == 'test' + - pytest-xdist>=3 ; extra == 'test' + - jaraco-path>=3.7.2 ; extra == 'test' + - build[virtualenv]>=1.0.3 ; extra == 'test' + - filelock>=3.4.0 ; extra == 'test' + - ini2toml[lite]>=0.14 ; extra == 'test' + - tomli-w>=1.0.0 ; extra == 'test' + - pytest-timeout ; extra == 'test' + - pytest-perf ; sys_platform != 'cygwin' and extra == 'test' + - jaraco-develop>=7.21 ; python_full_version >= '3.9' and sys_platform != 'cygwin' and extra == 'test' + - pytest-home>=0.5 ; extra == 'test' + - pytest-subprocess ; extra == 'test' + - pyproject-hooks!=1.1 ; extra == 'test' + - jaraco-test>=5.5 ; extra == 'test' + - sphinx>=3.5 ; extra == 'doc' + - jaraco-packaging>=9.3 ; extra == 'doc' + - rst-linker>=1.9 ; extra == 'doc' + - furo ; extra == 'doc' + - sphinx-lint ; extra == 'doc' + - jaraco-tidelift>=1.4 ; extra == 'doc' + - pygments-github-lexers==0.0.5 ; extra == 'doc' + - sphinx-favicon ; extra == 'doc' + - sphinx-inline-tabs ; extra == 'doc' + - sphinx-reredirects ; extra == 'doc' + - sphinxcontrib-towncrier ; extra == 'doc' + - sphinx-notfound-page>=1,<2 ; extra == 'doc' + - pyproject-hooks!=1.1 ; extra == 'doc' + - towncrier<24.7 ; extra == 'doc' + - packaging>=24.2 ; extra == 'core' + - more-itertools>=8.8 ; extra == 'core' + - jaraco-text>=3.7 ; extra == 'core' + - importlib-metadata>=6 ; python_full_version < '3.10' and extra == 'core' + - tomli>=2.0.1 ; python_full_version < '3.11' and extra == 'core' + - wheel>=0.43.0 ; extra == 'core' + - platformdirs>=4.2.2 ; extra == 'core' + - jaraco-functools>=4 ; extra == 'core' + - more-itertools ; extra == 'core' + - pytest-checkdocs>=2.4 ; extra == 'check' + - pytest-ruff>=0.2.1 ; sys_platform != 'cygwin' and extra == 'check' + - ruff>=0.13.0 ; sys_platform != 'cygwin' and extra == 'check' + - pytest-cov ; extra == 'cover' + - pytest-enabler>=2.2 ; extra == 'enabler' + - pytest-mypy ; extra == 'type' + - mypy==1.18.* ; extra == 'type' + - importlib-metadata>=7.0.2 ; python_full_version < '3.10' and extra == 'type' + - jaraco-develop>=7.21 ; sys_platform != 'cygwin' and extra == 'type' + requires_python: '>=3.9' - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-82.0.0-pyh332efcf_0.conda sha256: fd7201e38e38bf7f25818d624ca8da97b8998957ca9ae3fb7fdc9c17e6b25fcd md5: 1d00d46c634177fc8ede8b99d6089239 @@ -6408,6 +6816,11 @@ packages: - pkg:pypi/six?source=hash-mapping size: 18455 timestamp: 1753199211006 +- pypi: https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl + name: smmap + version: 5.0.2 + sha256: b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e + requires_python: '>=3.7' - conda: https://conda.anaconda.org/conda-forge/linux-64/snappy-1.2.2-h03e3b7b_1.conda sha256: 48f3f6a76c34b2cfe80de9ce7f2283ecb55d5ed47367ba91e8bb8104e12b8f11 md5: 98b6c9dc80eb87b2519b97bcf7e578dd @@ -6515,6 +6928,27 @@ packages: - blosc2>=2.3.0 - typing-extensions>=4.4.0 requires_python: '>=3.11' +- pypi: https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl + name: tensorboard + version: 2.20.0 + sha256: 9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6 + requires_dist: + - absl-py>=0.4 + - grpcio>=1.48.2 + - markdown>=2.6.8 + - numpy>=1.12.0 + - packaging + - pillow + - protobuf>=3.19.6,!=4.24.0 + - setuptools>=41.0.0 + - tensorboard-data-server>=0.7.0,<0.8.0 + - werkzeug>=1.0.1 + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl + name: tensorboard-data-server + version: 0.7.2 + sha256: 7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb + requires_python: '>=3.7' - conda: https://conda.anaconda.org/conda-forge/noarch/terminado-0.18.1-pyhc90fa1f_1.conda sha256: 6b6727a13d1ca6a23de5e6686500d0669081a117736a87c8abf444d60c1e40eb md5: 17b43cee5cc84969529d5d0b0309b2cb @@ -6750,6 +7184,156 @@ packages: version: 1.8.0 sha256: 2e911c2918603f945c26ff21a3a838d12709223dc4ccf243407bce8b6e897b46 requires_python: '>=3.7' +- pypi: https://files.pythonhosted.org/packages/02/21/aa0f434434c48490f91b65962b1ce863fdcce63febc166ca9fe9d706c2b6/torchmetrics-1.8.2-py3-none-any.whl + name: torchmetrics + version: 1.8.2 + sha256: 08382fd96b923e39e904c4d570f3d49e2cc71ccabd2a94e0f895d1f0dac86242 + requires_dist: + - numpy>1.20.0 + - packaging>17.1 + - torch>=2.0.0 + - lightning-utilities>=0.8.0 + - onnxruntime>=1.12.0 ; extra == 'audio' + - requests>=2.19.0 ; extra == 'audio' + - torchaudio>=2.0.1 ; extra == 'audio' + - gammatone>=1.0.0 ; extra == 'audio' + - pystoi>=0.4.0 ; extra == 'audio' + - pesq>=0.0.4 ; extra == 'audio' + - librosa>=0.10.0 ; extra == 'audio' + - torch-linear-assignment>=0.0.2 ; extra == 'clustering' + - pycocotools>2.0.0 ; extra == 'detection' + - torchvision>=0.15.1 ; extra == 'detection' + - torch-fidelity<=0.4.0 ; extra == 'image' + - torchvision>=0.15.1 ; extra == 'image' + - scipy>1.0.0 ; extra == 'image' + - piq<=0.8.0 ; extra == 'multimodal' + - einops>=0.7.0 ; extra == 'multimodal' + - transformers>=4.43.0 ; extra == 'multimodal' + - timm>=0.9.0 ; extra == 'multimodal' + - transformers>=4.43.0 ; extra == 'text' + - regex>=2021.9.24 ; extra == 'text' + - sentencepiece>=0.2.0 ; extra == 'text' + - nltk>3.8.1 ; extra == 'text' + - tqdm<4.68.0 ; extra == 'text' + - mecab-python3>=1.0.6 ; extra == 'text' + - ipadic>=1.0.0 ; extra == 'text' + - mypy==1.17.1 ; extra == 'typing' + - types-six ; extra == 'typing' + - torch==2.8.0 ; extra == 'typing' + - types-emoji ; extra == 'typing' + - types-protobuf ; extra == 'typing' + - types-setuptools ; extra == 'typing' + - types-requests ; extra == 'typing' + - types-tabulate ; extra == 'typing' + - types-pyyaml ; extra == 'typing' + - einops>=0.7.0 ; extra == 'video' + - vmaf-torch>=1.1.0 ; extra == 'video' + - scienceplots>=2.0.0 ; extra == 'visual' + - matplotlib>=3.6.0 ; extra == 'visual' + - onnxruntime>=1.12.0 ; extra == 'all' + - requests>=2.19.0 ; extra == 'all' + - torchaudio>=2.0.1 ; extra == 'all' + - gammatone>=1.0.0 ; extra == 'all' + - pystoi>=0.4.0 ; extra == 'all' + - pesq>=0.0.4 ; extra == 'all' + - librosa>=0.10.0 ; extra == 'all' + - torch-linear-assignment>=0.0.2 ; extra == 'all' + - pycocotools>2.0.0 ; extra == 'all' + - torchvision>=0.15.1 ; extra == 'all' + - torch-fidelity<=0.4.0 ; extra == 'all' + - torchvision>=0.15.1 ; extra == 'all' + - scipy>1.0.0 ; extra == 'all' + - piq<=0.8.0 ; extra == 'all' + - einops>=0.7.0 ; extra == 'all' + - transformers>=4.43.0 ; extra == 'all' + - timm>=0.9.0 ; extra == 'all' + - transformers>=4.43.0 ; extra == 'all' + - regex>=2021.9.24 ; extra == 'all' + - sentencepiece>=0.2.0 ; extra == 'all' + - nltk>3.8.1 ; extra == 'all' + - tqdm<4.68.0 ; extra == 'all' + - mecab-python3>=1.0.6 ; extra == 'all' + - ipadic>=1.0.0 ; extra == 'all' + - mypy==1.17.1 ; extra == 'all' + - types-six ; extra == 'all' + - torch==2.8.0 ; extra == 'all' + - types-emoji ; extra == 'all' + - types-protobuf ; extra == 'all' + - types-setuptools ; extra == 'all' + - types-requests ; extra == 'all' + - types-tabulate ; extra == 'all' + - types-pyyaml ; extra == 'all' + - einops>=0.7.0 ; extra == 'all' + - vmaf-torch>=1.1.0 ; extra == 'all' + - scienceplots>=2.0.0 ; extra == 'all' + - matplotlib>=3.6.0 ; extra == 'all' + - onnxruntime>=1.12.0 ; extra == 'dev' + - requests>=2.19.0 ; extra == 'dev' + - torchaudio>=2.0.1 ; extra == 'dev' + - gammatone>=1.0.0 ; extra == 'dev' + - pystoi>=0.4.0 ; extra == 'dev' + - pesq>=0.0.4 ; extra == 'dev' + - librosa>=0.10.0 ; extra == 'dev' + - torch-linear-assignment>=0.0.2 ; extra == 'dev' + - pycocotools>2.0.0 ; extra == 'dev' + - torchvision>=0.15.1 ; extra == 'dev' + - torch-fidelity<=0.4.0 ; extra == 'dev' + - torchvision>=0.15.1 ; extra == 'dev' + - scipy>1.0.0 ; extra == 'dev' + - piq<=0.8.0 ; extra == 'dev' + - einops>=0.7.0 ; extra == 'dev' + - transformers>=4.43.0 ; extra == 'dev' + - timm>=0.9.0 ; extra == 'dev' + - transformers>=4.43.0 ; extra == 'dev' + - regex>=2021.9.24 ; extra == 'dev' + - sentencepiece>=0.2.0 ; extra == 'dev' + - nltk>3.8.1 ; extra == 'dev' + - tqdm<4.68.0 ; extra == 'dev' + - mecab-python3>=1.0.6 ; extra == 'dev' + - ipadic>=1.0.0 ; extra == 'dev' + - mypy==1.17.1 ; extra == 'dev' + - types-six ; extra == 'dev' + - torch==2.8.0 ; extra == 'dev' + - types-emoji ; extra == 'dev' + - types-protobuf ; extra == 'dev' + - types-setuptools ; extra == 'dev' + - types-requests ; extra == 'dev' + - types-tabulate ; extra == 'dev' + - types-pyyaml ; extra == 'dev' + - einops>=0.7.0 ; extra == 'dev' + - vmaf-torch>=1.1.0 ; extra == 'dev' + - scienceplots>=2.0.0 ; extra == 'dev' + - matplotlib>=3.6.0 ; extra == 'dev' + - properscoring==0.1 ; extra == 'dev' + - mir-eval>=0.6 ; extra == 'dev' + - pytorch-msssim==1.0.0 ; extra == 'dev' + - scikit-image>=0.19.0 ; extra == 'dev' + - sacrebleu>=2.3.0 ; extra == 'dev' + - dists-pytorch==0.1 ; extra == 'dev' + - torch-complex<0.5.0 ; extra == 'dev' + - pytdc==0.4.1 ; (python_full_version < '3.10' and extra == 'dev') or (python_full_version < '3.12' and sys_platform == 'win32' and extra == 'dev') + - netcal>1.0.0 ; extra == 'dev' + - lpips<=0.1.4 ; extra == 'dev' + - jiwer>=2.3.0 ; extra == 'dev' + - fairlearn ; extra == 'dev' + - monai==1.4.0 ; extra == 'dev' + - statsmodels>0.13.5 ; extra == 'dev' + - mecab-ko-dic>=1.0.0 ; python_full_version < '3.12' and extra == 'dev' + - sewar>=0.4.4 ; extra == 'dev' + - mecab-ko>=1.0.0,<1.1.0 ; python_full_version < '3.12' and extra == 'dev' + - faster-coco-eval>=1.6.3 ; extra == 'dev' + - huggingface-hub<0.35 ; extra == 'dev' + - numpy<2.4.0 ; extra == 'dev' + - permetrics==2.0.0 ; extra == 'dev' + - bert-score==0.3.13 ; extra == 'dev' + - scipy>1.0.0 ; extra == 'dev' + - kornia>=0.6.7 ; extra == 'dev' + - rouge-score>0.1.0 ; extra == 'dev' + - fast-bss-eval>=0.1.0 ; extra == 'dev' + - aeon>=1.0.0 ; python_full_version >= '3.11' and extra == 'dev' + - pandas>1.4.0 ; extra == 'dev' + - dython==0.7.9 ; extra == 'dev' + requires_python: '>=3.9' - pypi: https://download.pytorch.org/whl/cpu/torchvision-0.25.0-cp311-cp311-macosx_11_0_arm64.whl name: torchvision version: 0.25.0 @@ -7149,6 +7733,13 @@ packages: purls: [] size: 91383 timestamp: 1756220668932 +- pypi: https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl + name: typing-inspection + version: 0.4.2 + sha256: 4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 + requires_dist: + - typing-extensions>=4.12.0 + requires_python: '>=3.9' - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda sha256: 032271135bca55aeb156cee361c81350c6f3fb203f57d024d7e5a1fc9ef18731 md5: 0caa1af407ecff61170c9437a808404d @@ -7281,6 +7872,213 @@ packages: purls: [] size: 115235 timestamp: 1767320173250 +- pypi: https://files.pythonhosted.org/packages/25/97/460f6cb738aaa39b4eb2e6b4c630b2ae4321cdd70a79d5955ea75a878981/wandb-0.25.0-py3-none-win_amd64.whl + name: wandb + version: 0.25.0 + sha256: 78307ac0b328f2dc334c8607bec772851215584b62c439eb320c4af4fb077a00 + requires_dist: + - click>=8.0.1 + - eval-type-backport ; python_full_version < '3.10' + - gitpython>=1.0.0,!=3.1.29 + - packaging + - platformdirs + - protobuf>=3.15.0,!=4.21.0,!=5.28.0,<7 ; python_full_version == '3.9.*' and sys_platform == 'linux' + - protobuf>=3.19.0,!=4.21.0,!=5.28.0,<7 ; python_full_version >= '3.10' and sys_platform == 'linux' + - protobuf>=3.19.0,!=4.21.0,!=5.28.0,<7 ; sys_platform != 'linux' + - pydantic<3 + - pyyaml + - requests>=2.0.0,<3 + - sentry-sdk>=2.0.0 + - typing-extensions>=4.8,<5 + - boto3 ; extra == 'aws' + - botocore>=1.5.76 ; extra == 'aws' + - azure-identity ; extra == 'azure' + - azure-storage-blob ; extra == 'azure' + - google-cloud-storage ; extra == 'gcp' + - filelock ; extra == 'importers' + - mlflow ; extra == 'importers' + - polars<=1.2.1 ; extra == 'importers' + - rich ; extra == 'importers' + - tenacity ; extra == 'importers' + - google-cloud-storage ; extra == 'kubeflow' + - kubernetes ; extra == 'kubeflow' + - minio ; extra == 'kubeflow' + - sh ; extra == 'kubeflow' + - awscli ; extra == 'launch' + - azure-containerregistry ; extra == 'launch' + - azure-identity ; extra == 'launch' + - azure-storage-blob ; extra == 'launch' + - boto3 ; extra == 'launch' + - botocore>=1.5.76 ; extra == 'launch' + - chardet ; extra == 'launch' + - google-auth ; extra == 'launch' + - google-cloud-aiplatform ; extra == 'launch' + - google-cloud-artifact-registry ; extra == 'launch' + - google-cloud-compute ; extra == 'launch' + - google-cloud-storage ; extra == 'launch' + - iso8601 ; extra == 'launch' + - jsonschema ; extra == 'launch' + - kubernetes ; extra == 'launch' + - kubernetes-asyncio ; extra == 'launch' + - nbconvert ; extra == 'launch' + - nbformat ; extra == 'launch' + - optuna ; extra == 'launch' + - pydantic ; extra == 'launch' + - pyyaml>=6.0.0 ; extra == 'launch' + - tomli ; extra == 'launch' + - tornado>=6.5.0 ; python_full_version >= '3.9' and extra == 'launch' + - typing-extensions ; extra == 'launch' + - bokeh ; extra == 'media' + - imageio>=2.28.1 ; extra == 'media' + - moviepy>=1.0.0 ; extra == 'media' + - numpy ; extra == 'media' + - pillow ; extra == 'media' + - plotly>=5.18.0 ; extra == 'media' + - rdkit ; extra == 'media' + - soundfile ; extra == 'media' + - cloudpickle ; extra == 'models' + - orjson ; extra == 'perf' + - sweeps>=0.2.0 ; extra == 'sweeps' + - wandb-workspaces ; extra == 'workspaces' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/c1/7d/0c131db3ec9deaabbd32263d90863cbfbe07659527e11c35a5c738cecdc5/wandb-0.25.0-py3-none-macosx_12_0_arm64.whl + name: wandb + version: 0.25.0 + sha256: 5eecb3c7b5e60d1acfa4b056bfbaa0b79a482566a9db58c9f99724b3862bc8e5 + requires_dist: + - click>=8.0.1 + - eval-type-backport ; python_full_version < '3.10' + - gitpython>=1.0.0,!=3.1.29 + - packaging + - platformdirs + - protobuf>=3.15.0,!=4.21.0,!=5.28.0,<7 ; python_full_version == '3.9.*' and sys_platform == 'linux' + - protobuf>=3.19.0,!=4.21.0,!=5.28.0,<7 ; python_full_version >= '3.10' and sys_platform == 'linux' + - protobuf>=3.19.0,!=4.21.0,!=5.28.0,<7 ; sys_platform != 'linux' + - pydantic<3 + - pyyaml + - requests>=2.0.0,<3 + - sentry-sdk>=2.0.0 + - typing-extensions>=4.8,<5 + - boto3 ; extra == 'aws' + - botocore>=1.5.76 ; extra == 'aws' + - azure-identity ; extra == 'azure' + - azure-storage-blob ; extra == 'azure' + - google-cloud-storage ; extra == 'gcp' + - filelock ; extra == 'importers' + - mlflow ; extra == 'importers' + - polars<=1.2.1 ; extra == 'importers' + - rich ; extra == 'importers' + - tenacity ; extra == 'importers' + - google-cloud-storage ; extra == 'kubeflow' + - kubernetes ; extra == 'kubeflow' + - minio ; extra == 'kubeflow' + - sh ; extra == 'kubeflow' + - awscli ; extra == 'launch' + - azure-containerregistry ; extra == 'launch' + - azure-identity ; extra == 'launch' + - azure-storage-blob ; extra == 'launch' + - boto3 ; extra == 'launch' + - botocore>=1.5.76 ; extra == 'launch' + - chardet ; extra == 'launch' + - google-auth ; extra == 'launch' + - google-cloud-aiplatform ; extra == 'launch' + - google-cloud-artifact-registry ; extra == 'launch' + - google-cloud-compute ; extra == 'launch' + - google-cloud-storage ; extra == 'launch' + - iso8601 ; extra == 'launch' + - jsonschema ; extra == 'launch' + - kubernetes ; extra == 'launch' + - kubernetes-asyncio ; extra == 'launch' + - nbconvert ; extra == 'launch' + - nbformat ; extra == 'launch' + - optuna ; extra == 'launch' + - pydantic ; extra == 'launch' + - pyyaml>=6.0.0 ; extra == 'launch' + - tomli ; extra == 'launch' + - tornado>=6.5.0 ; python_full_version >= '3.9' and extra == 'launch' + - typing-extensions ; extra == 'launch' + - bokeh ; extra == 'media' + - imageio>=2.28.1 ; extra == 'media' + - moviepy>=1.0.0 ; extra == 'media' + - numpy ; extra == 'media' + - pillow ; extra == 'media' + - plotly>=5.18.0 ; extra == 'media' + - rdkit ; extra == 'media' + - soundfile ; extra == 'media' + - cloudpickle ; extra == 'models' + - orjson ; extra == 'perf' + - sweeps>=0.2.0 ; extra == 'sweeps' + - wandb-workspaces ; extra == 'workspaces' + requires_python: '>=3.9' +- pypi: https://files.pythonhosted.org/packages/de/91/ec9465d014cfd199c5b2083d271d31b3c2aedeae66f3d8a0712f7f54bdf3/wandb-0.25.0-py3-none-manylinux_2_28_x86_64.whl + name: wandb + version: 0.25.0 + sha256: 6c4c38077836f9b7569a35b0e1dcf1f0c43616fcd936d182f475edbfea063665 + requires_dist: + - click>=8.0.1 + - eval-type-backport ; python_full_version < '3.10' + - gitpython>=1.0.0,!=3.1.29 + - packaging + - platformdirs + - protobuf>=3.15.0,!=4.21.0,!=5.28.0,<7 ; python_full_version == '3.9.*' and sys_platform == 'linux' + - protobuf>=3.19.0,!=4.21.0,!=5.28.0,<7 ; python_full_version >= '3.10' and sys_platform == 'linux' + - protobuf>=3.19.0,!=4.21.0,!=5.28.0,<7 ; sys_platform != 'linux' + - pydantic<3 + - pyyaml + - requests>=2.0.0,<3 + - sentry-sdk>=2.0.0 + - typing-extensions>=4.8,<5 + - boto3 ; extra == 'aws' + - botocore>=1.5.76 ; extra == 'aws' + - azure-identity ; extra == 'azure' + - azure-storage-blob ; extra == 'azure' + - google-cloud-storage ; extra == 'gcp' + - filelock ; extra == 'importers' + - mlflow ; extra == 'importers' + - polars<=1.2.1 ; extra == 'importers' + - rich ; extra == 'importers' + - tenacity ; extra == 'importers' + - google-cloud-storage ; extra == 'kubeflow' + - kubernetes ; extra == 'kubeflow' + - minio ; extra == 'kubeflow' + - sh ; extra == 'kubeflow' + - awscli ; extra == 'launch' + - azure-containerregistry ; extra == 'launch' + - azure-identity ; extra == 'launch' + - azure-storage-blob ; extra == 'launch' + - boto3 ; extra == 'launch' + - botocore>=1.5.76 ; extra == 'launch' + - chardet ; extra == 'launch' + - google-auth ; extra == 'launch' + - google-cloud-aiplatform ; extra == 'launch' + - google-cloud-artifact-registry ; extra == 'launch' + - google-cloud-compute ; extra == 'launch' + - google-cloud-storage ; extra == 'launch' + - iso8601 ; extra == 'launch' + - jsonschema ; extra == 'launch' + - kubernetes ; extra == 'launch' + - kubernetes-asyncio ; extra == 'launch' + - nbconvert ; extra == 'launch' + - nbformat ; extra == 'launch' + - optuna ; extra == 'launch' + - pydantic ; extra == 'launch' + - pyyaml>=6.0.0 ; extra == 'launch' + - tomli ; extra == 'launch' + - tornado>=6.5.0 ; python_full_version >= '3.9' and extra == 'launch' + - typing-extensions ; extra == 'launch' + - bokeh ; extra == 'media' + - imageio>=2.28.1 ; extra == 'media' + - moviepy>=1.0.0 ; extra == 'media' + - numpy ; extra == 'media' + - pillow ; extra == 'media' + - plotly>=5.18.0 ; extra == 'media' + - rdkit ; extra == 'media' + - soundfile ; extra == 'media' + - cloudpickle ; extra == 'models' + - orjson ; extra == 'perf' + - sweeps>=0.2.0 ; extra == 'sweeps' + - wandb-workspaces ; extra == 'workspaces' + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl name: wcwidth version: 0.6.0 @@ -7330,6 +8128,14 @@ packages: - pkg:pypi/websocket-client?source=hash-mapping size: 61391 timestamp: 1759928175142 +- pypi: https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl + name: werkzeug + version: 3.1.6 + sha256: 7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131 + requires_dist: + - markupsafe>=2.1.1 + - watchdog>=2.3 ; extra == 'watchdog' + requires_python: '>=3.9' - pypi: https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl name: widgetsnbextension version: 4.0.15 diff --git a/pyproject.toml b/pyproject.toml index 6bd4cf5..21413d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,11 +50,15 @@ torchvision = { version = ">=0.20.1", index = "https://download.pytorch.org/whl/ torch = { version = ">=2.5.1", index = "https://download.pytorch.org/whl/cpu" } torchvision = { version = ">=0.20.1", index = "https://download.pytorch.org/whl/cpu" } +[tool.ruff] +line-length = 88 + [tool.pixi.tasks] [tool.pixi.dependencies] python = ">=3.11,<3.12" hydra-core = ">=1.3.2,<2" +line_profiler = ">=5.0.2,<6" [tool.pixi.feature.fdp] platforms = ["linux-64"] diff --git a/scripts/data_fetching_omega/read_mds.sh b/scripts/data_fetching_omega/read_mds.sh index 0b0dda7..4830336 100644 --- a/scripts/data_fetching_omega/read_mds.sh +++ b/scripts/data_fetching_omega/read_mds.sh @@ -26,135 +26,162 @@ fi echo "=========================================" echo "Job started at: $(date)" echo "Shot number: ${SHOT_NUMBER}" -echo "Config file: ${CONFIG_FILE}" +echo "Config files: ${CONFIG_FILES}" echo "Chunk size: ${CHUNK_SIZE}" echo "=========================================" OUTPUT_FILE="${OUTPUT_DIR}/${SHOT_NUMBER}.h5" +TOTAL_FAILED_CHUNKS=0 -# Extract server -SERVER=$(grep "^server:" ${CONFIG_FILE} | cut -d: -f2- | xargs) - -# Create flat list: each line is "tree_name|signal_line" -TMP_FLAT_LIST=$(mktemp) - -awk ' -/^ [a-z0-9_]+:$/ { - current_tree = $1 - sub(/:$/, "", current_tree) - next -} -/^ - / { - if (current_tree != "") { - print current_tree "|" $0 +# Process each config file sequentially +for CONFIG_FILE in ${CONFIG_FILES}; do + echo "" + echo "=========================================" + echo "Processing config: ${CONFIG_FILE}" + echo "=========================================" + + if [ ! -f "${CONFIG_FILE}" ]; then + echo "ERROR: Config file not found: ${CONFIG_FILE}" + TOTAL_FAILED_CHUNKS=$((TOTAL_FAILED_CHUNKS + 1)) + continue + fi + + # Extract server + SERVER=$(grep "^server:" ${CONFIG_FILE} | cut -d: -f2- | xargs) + echo "Server: ${SERVER}" + + # Create flat list: each line is "tree_name|signal_line" + TMP_FLAT_LIST=$(mktemp) + + awk ' + /^ [a-zA-Z0-9_]+:$/ { + current_tree = $1 + sub(/:$/, "", current_tree) + next + } + /^ - / { + if (current_tree != "") { + print current_tree "|" $0 + } } -} -' ${CONFIG_FILE} > ${TMP_FLAT_LIST} + ' ${CONFIG_FILE} > ${TMP_FLAT_LIST} -TOTAL_SIGNALS=$(wc -l < ${TMP_FLAT_LIST}) -NUM_CHUNKS=$(( (TOTAL_SIGNALS + CHUNK_SIZE - 1) / CHUNK_SIZE )) + TOTAL_SIGNALS=$(wc -l < ${TMP_FLAT_LIST}) + NUM_CHUNKS=$(( (TOTAL_SIGNALS + CHUNK_SIZE - 1) / CHUNK_SIZE )) -echo "Total signals: ${TOTAL_SIGNALS}" -echo "Processing in ${NUM_CHUNKS} chunks" -echo "=========================================" + echo "Total signals: ${TOTAL_SIGNALS}" + echo "Processing in ${NUM_CHUNKS} chunks" + echo "=========================================" -FAILED_CHUNKS=0 + FAILED_CHUNKS=0 -for (( chunk=0; chunk "${CONFIG_FILE_CHUNK}" << EOF + cat > "${CONFIG_FILE_CHUNK}" << EOF shot_numbers: - ${SHOT_NUMBER} trees: EOF - # Group signals by tree and add to config - echo "${CHUNK_DATA}" | awk -F'|' ' - { - tree = $1 - signal = $2 - if (tree != current_tree) { - if (current_tree != "") { - # Print accumulated signals for previous tree - for (i = 0; i < sig_count; i++) { - print signals[i] + # Group signals by tree and add to config + echo "${CHUNK_DATA}" | awk -F'|' ' + { + tree = $1 + signal = $2 + if (tree != current_tree) { + if (current_tree != "") { + # Print accumulated signals for previous tree + for (i = 0; i < sig_count; i++) { + print signals[i] + } } + # Start new tree + current_tree = tree + print " " tree ":" + sig_count = 0 } - # Start new tree - current_tree = tree - print " " tree ":" - sig_count = 0 + signals[sig_count++] = signal } - signals[sig_count++] = signal - } - END { - # Print last tree signals - if (sig_count > 0) { - for (i = 0; i < sig_count; i++) { - print signals[i] + END { + # Print last tree signals + if (sig_count > 0) { + for (i = 0; i < sig_count; i++) { + print signals[i] + } } } - } - ' >> "${CONFIG_FILE_CHUNK}" + ' >> "${CONFIG_FILE_CHUNK}" - # Add output file and server - cat >> "${CONFIG_FILE_CHUNK}" << EOF + # Add output file and server + cat >> "${CONFIG_FILE_CHUNK}" << EOF out_filename: ${OUTPUT_FILE} server: ${SERVER} EOF - # Run read_mds - echo " Running read_mds..." - read_mds -c ${CONFIG_FILE_CHUNK} - EXIT_CODE=$? + # Run read_mds + echo " Running read_mds..." + read_mds -c ${CONFIG_FILE_CHUNK} + EXIT_CODE=$? - if [ ${EXIT_CODE} -eq 0 ]; then - echo " ✓ Chunk ${CHUNK_NUM}/${NUM_CHUNKS} completed successfully" - rm -f ${CONFIG_FILE_CHUNK} - else - echo " ✗ Chunk ${CHUNK_NUM}/${NUM_CHUNKS} FAILED (exit code: ${EXIT_CODE})" - echo " Config preserved: ${CONFIG_FILE_CHUNK}" - FAILED_CHUNKS=$((FAILED_CHUNKS + 1)) - fi -done + if [ ${EXIT_CODE} -eq 0 ]; then + echo " ✓ Chunk ${CHUNK_NUM}/${NUM_CHUNKS} completed successfully" + rm -f ${CONFIG_FILE_CHUNK} + else + echo " ✗ Chunk ${CHUNK_NUM}/${NUM_CHUNKS} FAILED (exit code: ${EXIT_CODE})" + echo " Config preserved: ${CONFIG_FILE_CHUNK}" + FAILED_CHUNKS=$((FAILED_CHUNKS + 1)) + fi + done + + rm -f ${TMP_FLAT_LIST} -rm -f ${TMP_FLAT_LIST} + echo "" + echo "=========================================" + echo "Config ${CONFIG_FILE} summary:" + echo " Total signals: ${TOTAL_SIGNALS}" + echo " Total chunks: ${NUM_CHUNKS}" + echo " Failed chunks: ${FAILED_CHUNKS}" + echo "=========================================" + + TOTAL_FAILED_CHUNKS=$((TOTAL_FAILED_CHUNKS + FAILED_CHUNKS)) +done +# Overall summary echo "" echo "=========================================" -echo "Processing summary:" -echo " Total signals: ${TOTAL_SIGNALS}" -echo " Total chunks: ${NUM_CHUNKS}" -echo " Failed chunks: ${FAILED_CHUNKS}" +echo "Overall processing summary for shot ${SHOT_NUMBER}:" +echo " Configs processed: ${CONFIG_FILES}" +echo " Total failed chunks: ${TOTAL_FAILED_CHUNKS}" echo "=========================================" # Check overall success -if [ ${FAILED_CHUNKS} -eq 0 ]; then +if [ ${TOTAL_FAILED_CHUNKS} -eq 0 ]; then if [ -f "${OUTPUT_FILE}" ] && [ -s "${OUTPUT_FILE}" ]; then - echo "SUCCESS: All chunks completed, output file: ${OUTPUT_FILE}" + echo "SUCCESS: All configs completed, output file: ${OUTPUT_FILE}" ( flock -x 200 @@ -171,15 +198,9 @@ if [ ${FAILED_CHUNKS} -eq 0 ]; then echo "=========================================" echo "Starting Globus transfer..." - # Get relative path of the output file OUTPUT_FILENAME=$(basename "${OUTPUT_FILE}") - - # Strip /cscratch/ from the path for Globus - # If OUTPUT_FILE="/cscratch/steinerp/database/data/170659.h5" - # Then GLOBUS_SOURCE_PATH="steinerp/database/data/170659.h5" GLOBUS_SOURCE_PATH="${OUTPUT_FILE#/cscratch/}" - # Transfer this file echo "Transferring: ${OUTPUT_FILENAME}" echo "Source path: ${GLOBUS_SOURCE_PATH}" echo "Dest path: ${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}" @@ -189,7 +210,7 @@ if [ ${FAILED_CHUNKS} -eq 0 ]; then --label "Auto-transfer ${OUTPUT_FILENAME} $(date +%Y%m%d-%H%M%S)" \ --jmespath 'task_id' \ --format unix \ - --notify off \ + --notify off \ "${GLOBUS_SOURCE_ENDPOINT}:${GLOBUS_SOURCE_PATH}" \ "${GLOBUS_DEST_ENDPOINT}:${GLOBUS_DEST_PATH}${OUTPUT_FILENAME}") @@ -200,20 +221,17 @@ if [ ${FAILED_CHUNKS} -eq 0 ]; then echo "Transfer submitted: Task ID ${TRANSFER_TASK_ID}" echo "Waiting for transfer to complete..." - # Wait for transfer (with 2 hour timeout) globus task wait "${TRANSFER_TASK_ID}" --timeout 7200 --polling-interval 30 if [ $? -eq 0 ]; then echo "✓ Transfer completed successfully!" echo "Deleting local file to free up space..." - # Delete the transferred file rm -f "${OUTPUT_FILE}" if [ $? -eq 0 ]; then echo "✓ Local file deleted: ${OUTPUT_FILE}" - # Log the transfer TRANSFER_LOG="${OUTPUT_DIR}/globus_transfers.log" echo "$(date '+%Y-%m-%d %H:%M:%S') | ${SHOT_NUMBER} | ${OUTPUT_FILENAME} | TRANSFERRED_AND_DELETED" >> ${TRANSFER_LOG} else @@ -230,12 +248,12 @@ if [ ${FAILED_CHUNKS} -eq 0 ]; then echo "=========================================" else echo "" - echo "=========================================" - echo "Globus transfer disabled - file retained locally" - echo "File location: ${OUTPUT_FILE}" - echo "=========================================" + echo "=========================================" + echo "Globus transfer disabled - file retained locally" + echo "File location: ${OUTPUT_FILE}" + echo "=========================================" fi - # ============================================ + # ============================================ # END GLOBUS TRANSFER SECTION # ============================================ @@ -243,11 +261,11 @@ if [ ${FAILED_CHUNKS} -eq 0 ]; then exit 0 else echo "ERROR: Output file missing or empty: ${OUTPUT_FILE}" - FAILED_CHUNKS=1 + TOTAL_FAILED_CHUNKS=1 fi fi -echo "ERROR: ${FAILED_CHUNKS} chunk(s) failed for shot ${SHOT_NUMBER}" +echo "ERROR: ${TOTAL_FAILED_CHUNKS} chunk(s) failed for shot ${SHOT_NUMBER}" ( flock -x 200 diff --git a/scripts/data_fetching_omega/submit_read_mds_batches.sh b/scripts/data_fetching_omega/submit_read_mds_batches.sh index bec9efa..5991312 100644 --- a/scripts/data_fetching_omega/submit_read_mds_batches.sh +++ b/scripts/data_fetching_omega/submit_read_mds_batches.sh @@ -14,7 +14,7 @@ SHOT_END=200800 SHOT_LIST_FILE="shots_to_process.txt" # Common configuration -CONFIG_FILE="config_atlas.yaml" +CONFIG_FILES="config_atlas.yaml config_chiron.yaml" # Process both servers OUTPUT_DIR="/cscratch/steinerp/database/data" NODE_PATHS_DIR="/cscratch/steinerp/database/node_paths" # Deprecated but kept for compatibility @@ -43,7 +43,7 @@ echo "=========================================" echo "MDSPlus Batch Data Fetcher" echo "=========================================" echo "Mode: ${MODE}" -echo "Config file: ${CONFIG_FILE}" +echo "Config files: ${CONFIG_FILES}" if [ "${MODE}" = "range" ]; then echo "Shot range: ${SHOT_START} to ${SHOT_END}" @@ -54,6 +54,14 @@ else exit 1 fi +# Verify all config files exist +for config in ${CONFIG_FILES}; do + if [ ! -f "${config}" ]; then + echo "ERROR: Config file not found: ${config}" + exit 1 + fi +done + echo "Output directory: ${OUTPUT_DIR}" echo "Batch size: ${BATCH_SIZE}" echo "Max concurrent jobs: ${MAX_SUBMIT_LIMIT}" @@ -143,7 +151,7 @@ while [ ${SHOT_INDEX} -lt ${TOTAL_SHOTS} ]; do --array=1-${BATCH_SHOTS} \ --output=jobs/job_%A_%a.out \ --error=jobs/job_%A_%a.err \ - --export=ALL,BATCH_FILE=${BATCH_FILE},CONFIG_FILE=${CONFIG_FILE},OUTPUT_DIR=${OUTPUT_DIR},NODE_PATHS_DIR=${NODE_PATHS_DIR},COMPLETED_FILE=${COMPLETED_FILE},FAILED_FILE=${FAILED_FILE} \ + --export=ALL,BATCH_FILE=${BATCH_FILE},CONFIG_FILES="${CONFIG_FILES}",OUTPUT_DIR=${OUTPUT_DIR},NODE_PATHS_DIR=${NODE_PATHS_DIR},COMPLETED_FILE=${COMPLETED_FILE},FAILED_FILE=${FAILED_FILE} \ read_mds.sh) echo "Submitted batch ${BATCH_NUM} as job ${JOB_ID}" diff --git a/scripts/data_preparation/make_processing_stats.py b/scripts/data_preparation/make_processing_stats.py index 043bc56..f95b63b 100644 --- a/scripts/data_preparation/make_processing_stats.py +++ b/scripts/data_preparation/make_processing_stats.py @@ -5,7 +5,7 @@ def main(): hdf5_files = sorted( - Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("20000*_processed.h5") + Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("*_processed.h5") ) all_input_signals = [ @@ -32,7 +32,7 @@ def main(): max_duration_s=10., ) - compute_preprocessing_stats(dataset, 'preprocessing_stats_tmp.pt') + compute_preprocessing_stats(dataset, 'preprocessing_stats.pt') if __name__ == "__main__": diff --git a/scripts/data_preparation/prepare_data.py b/scripts/data_preparation/prepare_data.py index c7ef8f7..15a1c82 100644 --- a/scripts/data_preparation/prepare_data.py +++ b/scripts/data_preparation/prepare_data.py @@ -74,6 +74,9 @@ def load_signal_data( shot_group = self.h5_file[self.shot_number] + if tree not in shot_group: + tree = tree.lower() + if tree not in shot_group: if self.verbose: warnings.warn( diff --git a/scripts/slurm/prepare_data.sh b/scripts/slurm/prepare_data.sh index f1e2577..f684742 100755 --- a/scripts/slurm/prepare_data.sh +++ b/scripts/slurm/prepare_data.sh @@ -5,7 +5,7 @@ #SBATCH --cpus-per-task=32 # cpu-cores per task (>1 if multi-threaded tasks) #SBATCH --nodes=1 # node count #SBATCH --mem-per-cpu=16G # memory per cpu-core (4G is default) -#SBATCH --time=1:00:00 # total run time limit (HH:MM:SS) +#SBATCH --time=4:00:00 # total run time limit (HH:MM:SS) #SBATCH --mail-type=all # send email on job start, end and fault #SBATCH --mail-user=ps9551@princeton.edu diff --git a/scripts/slurm/train_filterscopes.sh b/scripts/slurm/train_filterscopes.sh new file mode 100644 index 0000000..1b111c7 --- /dev/null +++ b/scripts/slurm/train_filterscopes.sh @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --job-name=fast_time_series_reconstruction +#SBATCH --output=logs/%j_fast_time_series_reconstruction.out +#SBATCH --error=logs/%j_fast_time_series_reconstruction.err +#SBATCH --time=04:00:00 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=17 +#SBATCH --mem-per-cpu=8G + +export OMP_NUM_THREADS=1 +export PYTHONUNBUFFERED=1 + +srun pixi run python ../training/fast_time_series_reconstruction.py \ + --signal "filterscopes" \ + --d_model 512 \ + --batch_size 2048 \ + --num_workers 16 \ + --epochs 200 \ + --lr 1e-2 \ + --weight_decay 0.05 \ + --warmup_epochs 5 \ + --min_lr 0.0 \ + --checkpoint_dir runs \ + --stats_path /scratch/gpfs/ps9551/FusionAIHub/scripts/slurm/preprocessing_stats.pt diff --git a/scripts/training/fast_time_series_reconstruction.py b/scripts/training/fast_time_series_reconstruction.py index 808037d..c58190b 100644 --- a/scripts/training/fast_time_series_reconstruction.py +++ b/scripts/training/fast_time_series_reconstruction.py @@ -5,10 +5,9 @@ import torch import torch.nn as nn import torch.optim as optim -from torch.utils.data import ConcatDataset, DataLoader -from tokamak_foundation_model.data.data_loader import TokamakH5Dataset, collate_fn -from tokamak_foundation_model.data.utils import worker_init_fn +from tokamak_foundation_model.data.multi_file_dataset import ( + TokamakMultiFileDataset, make_dataloader) from tokamak_foundation_model.trainer.trainer import UnimodalTrainer from tokamak_foundation_model.models.model_factory import ( build_model, MODEL_REGISTRY, SIGNAL_MODEL_DEFAULTS) @@ -23,12 +22,13 @@ def main(): - ### Settings ### - parser = argparse.ArgumentParser(description="Train a unimodal autoencoder") + parser = argparse.ArgumentParser( + description="Train a unimodal autoencoder" + ) parser.add_argument( "--signal", choices=list(SIGNAL_MODEL_DEFAULTS.keys()), - default="d_alpha", + default="filterscopes", help="Signal name to train on" ) parser.add_argument( @@ -38,17 +38,20 @@ def main(): "--hop_length", type=int, default=256, help="Hop length for STFT.", ) parser.add_argument( - "--model", choices=list(MODEL_REGISTRY.keys()), default="fast_time_series", + "--model", + choices=list(MODEL_REGISTRY.keys()), + default="fast_time_series", help="Model type (default: auto-selected from signal)" ) parser.add_argument( "--data_dir", type=str, - default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/", + default="/scratch/gpfs/EKOLEMEN/foundation_model/", help="Path to HDF5 data directory" ) parser.add_argument( - "--stats_path", type=str, - default="C:/Users/admin/PycharmProjects/FusionAIHub/scripts/preprocessing_stats.pt", + "--stats_path", + type=str, + default="/scratch/gpfs/ps9551/FusionAIHub/scripts/slurm/preprocessing_stats.pt", help="Path to preprocessing stats file" ) parser.add_argument( @@ -59,12 +62,21 @@ def main(): help="Number of latent tokens (default: use model default)" ) parser.add_argument( - "--batch_size", type=int, default=2, - help="Batch size (for spectrograms, each sample's C channels are processed " - "independently, so effective batch = batch_size * C)" + "--batch_size", type=int, default=32, + help="Batch size (for spectrograms, each sample's C channels are " + "processed independently, so effective batch = batch_size * C)" + ) + parser.add_argument( + "--num_workers", + type=int, + default=4, + help="Number of data loader workers" ) parser.add_argument( - "--num_workers", type=int, default=4, help="Number of data loader workers" + "--prefetch_factor", + type=int, + default=4, + help="Batches to prefetch per worker" ) parser.add_argument( "--epochs", type=int, default=50, help="Number of training epochs" @@ -80,10 +92,13 @@ def main(): help="LR warmup epochs (0 to disable scheduler)" ) parser.add_argument( - "--min_lr", type=float, default=0.0, help="Minimum LR at end of cosine decay" + "--min_lr", type=float, default=0.0, + help="Minimum LR at end of cosine decay" ) parser.add_argument( - "--checkpoint_dir", type=str, default="runs", help="Directory for checkpoints" + "--checkpoint_dir", type=str, + default="/scratch/gpfs/ps9551/FusionAIHub/scripts/slurm/runs", + help="Directory for checkpoints" ) parser.add_argument( "--num_plots", type=int, default=4, @@ -112,25 +127,21 @@ def main(): ### Dataset Setup ### hdf5_files = sorted(data_dir.glob("*_processed.h5")) - stats = torch.load(statistics_path) - - datasets_processed = [ - TokamakH5Dataset( - hdf5_path=str(f), - preprocessing_stats=stats, - input_signals=[signal_name], - target_signals=[signal_name], - n_fft=args.n_fft, - hop_length=args.hop_length, - prediction_mode=False, - ) - for f in hdf5_files - ] - - concatenated_dataset = ConcatDataset(datasets_processed) + stats = torch.load(statistics_path, weights_only=False) + + dataset_processed = TokamakMultiFileDataset( + hdf5_paths=hdf5_files, + input_signals=[signal_name], + target_signals=[signal_name], + n_fft=args.n_fft, + hop_length=args.hop_length, + preprocessing_stats=stats, + prediction_mode=False, + lengths_cache_path="../slurm/dataset_lengths.pt", + ) # Not sure if this is elegant - sample_data = next(iter(concatenated_dataset))[signal_name] + sample_data = next(iter(dataset_processed))[signal_name] n_channels = sample_data.shape[0] logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") @@ -154,28 +165,25 @@ def main(): loss_fn = nn.L1Loss() - dataloader = DataLoader( - concatenated_dataset, + dataloader = make_dataloader( + dataset_processed, batch_size=args.batch_size, - collate_fn=collate_fn, - worker_init_fn=worker_init_fn, num_workers=args.num_workers, - persistent_workers=args.num_workers > 0, - pin_memory=True, shuffle=True, + pin_memory=True, + prefetch_factor=args.prefetch_factor, ) ### Training ### - drawer = DefaultDrawer(num_plots=args.num_plots) + drawer = DefaultDrawer() trainer = UnimodalTrainer( epochs=args.epochs, - checkpoint_path=checkpoint_path, model=model, - optimizer=optimizer, - lr_scheduler=lr_scheduler, loss_fn=loss_fn, - device=device, - drawer=drawer, + optimizer=optimizer, + scheduler=lr_scheduler, + checkpoint_path=checkpoint_path, + drawer=None, # drawer, log_interval=args.log_interval, ) @@ -183,7 +191,7 @@ def main(): logger.info(f"Resuming training from checkpoint: {checkpoint_path}") trainer.load_checkpoint(checkpoint_path=checkpoint_path) - trainer.train(dataloader, modality_key=signal_name) + trainer.fit(dataloader, modality_key=signal_name) if __name__ == "__main__": diff --git a/src/tokamak_foundation_model/data/config/config.yaml b/src/tokamak_foundation_model/data/config/config.yaml index b8266b3..9585910 100644 --- a/src/tokamak_foundation_model/data/config/config.yaml +++ b/src/tokamak_foundation_model/data/config/config.yaml @@ -1,6 +1,6 @@ defaults: - modalities: modalities - - shot_list: train_small + - shot_list: train_additional # These can be overridden from CLI, e.g.: # python generate_data.py shot_list=train diff --git a/src/tokamak_foundation_model/data/data_loader.py b/src/tokamak_foundation_model/data/data_loader.py index 355684e..7986662 100644 --- a/src/tokamak_foundation_model/data/data_loader.py +++ b/src/tokamak_foundation_model/data/data_loader.py @@ -1,377 +1,12 @@ import torch -from torch.utils.data import Dataset, DataLoader +from torch.utils.data import Dataset import numpy as np -import h5py +import h5py # type: ignore from pathlib import Path -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Optional import torch.nn.functional as F import copy -from line_profiler import profile - - -class WelfordTensor: - """ - Online Welford algorithm for per-channel statistics on batched tensors. - - Accumulates running mean, variance, minimum, and maximum over an arbitrary - number of :meth:`update` calls without storing the full dataset in memory. - Statistics are computed along the channel axis (axis 1 for 3-D and 4-D - tensors) by aggregating across the batch dimension and all remaining - non-channel dimensions. Batches that contain any ``NaN`` value are - silently skipped. - - The shape of the statistics vectors depends on the input rank: - - ========= =================================== =========== - ``ndim`` Interpretation Stats shape - ========= =================================== =========== - 4 ``(B, C, F, T)`` — spectrograms / ``(C,)`` - time series - 3 ``(B, S, T)`` — profiles ``(S,)`` - ≤ 2 ``(B, T)`` or scalar — video / ``(1,)`` - fallback - ========= =================================== =========== - - Attributes - ---------- - mean : torch.Tensor or None - Running per-channel mean, shape ``(C,)``. ``None`` before the first - :meth:`update` call. - std : torch.Tensor or None - Per-channel sample standard deviation, shape ``(C,)``. Populated - only after :meth:`compute` is called. - min_val : torch.Tensor or None - Running per-channel minimum, shape ``(C,)``. ``None`` before the - first :meth:`update` call. - max_val : torch.Tensor or None - Running per-channel maximum, shape ``(C,)``. ``None`` before the - first :meth:`update` call. - n : int - Total number of scalar samples seen so far (summed over all - non-channel dimensions across all batches). - M2 : torch.Tensor or None - Running sum of squared deviations from the mean (Welford - accumulator), shape ``(C,)``. ``None`` before the first - :meth:`update` call. - initialized : bool - ``True`` once the internal buffers have been allocated on the first - :meth:`update` call. - - Notes - ----- - The parallel (batch) variant of Welford's algorithm is used to combine - each incoming batch with the accumulated state in a single pass - [1]_. All accumulation is done in ``float64`` regardless of the input - dtype to minimise floating-point cancellation errors. - - References - ---------- - .. [1] Welford, B. P. (1962). Note on a method for calculating corrected - sums of squares and products. *Technometrics*, 4(3), 419–420. - https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm - - Examples - -------- - >>> import torch - >>> tracker = WelfordTensor() - >>> for _ in range(10): - ... batch = torch.randn(32, 8, 512, 200) # (B, C, F, T) - ... tracker.update(batch) - >>> stats = tracker.compute() - >>> stats['mean'].shape - (8,) - """ - - def __init__(self): - self.mean = None - self.std = None - self.min_val = None - self.max_val = None - self.n = 0 - self.M2 = None - self.initialized = False - - def _initialize(self, value: torch.Tensor): - """ - Allocate accumulator buffers sized to match *value*. - - Called automatically by :meth:`update` on the first non-NaN batch. - Derives the number of channels from the input rank: - - * ``ndim == 4``: channel axis is 1 (spectrograms / time series). - * ``ndim == 3``: channel axis is 1 (profiles / spatial signals). - * ``ndim <= 2``: treated as single-channel (``n_channels = 1``). - - Parameters - ---------- - value : torch.Tensor - First batch tensor, used only to infer ``n_channels``. - Shape must be ``(B, C, ...)`` for 3-D or 4-D inputs. - - Returns - ------- - None - """ - # Determine number of channels based on tensor shape (excluding batch dim) - if value.ndim == 4: - # (batch, channels, freq_bins, time) or (batch, channels, 1, time) - n_channels = value.shape[1] - elif value.ndim == 3: - # (batch, spatial_points, time) or (batch, time, height) - ambiguous - # Assume spatial/channel dim is second - n_channels = value.shape[1] - elif value.ndim == 2: - # (batch, time) - single channel - n_channels = 1 - else: - # Shouldn't happen, but treat as single channel - n_channels = 1 - - self.mean = torch.zeros(n_channels, dtype=torch.float64) - self.M2 = torch.zeros(n_channels, dtype=torch.float64) - self.min_val = torch.full( - (n_channels,), float('inf'), dtype=torch.float64) - self.max_val = torch.full( - (n_channels,), float('-inf'), dtype=torch.float64) - self.initialized = True - - def update(self, value: torch.Tensor): - """ - Incorporate a new batch into the running statistics. - - Batches that contain any ``NaN`` element are silently skipped. On - the first valid call the accumulator buffers are allocated via - :meth:`_initialize`. Subsequent calls merge the incoming batch - statistics with the accumulated state using the parallel Welford - update rule. - - Parameters - ---------- - value : torch.Tensor - Batched input tensor. Supported shapes: - - * ``(B, C, F, T)`` — spectrograms or multi-channel time series. - * ``(B, C, 1, T)`` — single-frequency time series. - * ``(B, S, T)`` — spatial profiles. - * ``(B, T, H, W)`` — video frames (global statistics). - - Returns - ------- - None - """ - # Skip if contains NaN - if torch.isnan(value).any(): - return - - # Initialize on first call - if not self.initialized: - self._initialize(value) - - # Convert to float64 for numerical stability - value = value.to(dtype=torch.float64) - - # Compute per-channel statistics by flattening batch - # and all non-channel dims - if value.ndim == 4 and value.shape[1] == self.mean.shape[0]: - # (batch, channels, freq_bins, time) → flatten batch, freq, time - # (B, C, F, T) → (C, B*F*T) - n_channels = value.shape[1] - value_flat = value.permute(1, 0, 2, 3).reshape(n_channels, -1) - - # Per-channel mean, min, max - batch_mean = value_flat.mean(dim=1) - batch_min = value_flat.min(dim=1).values - batch_max = value_flat.max(dim=1).values - n_samples = value_flat.shape[1] - - # For variance, we need sum of squared deviations - batch_var = value_flat.var(dim=1, unbiased=False) - batch_M2 = batch_var * n_samples - - elif value.ndim == 3: - # (batch, spatial_points, time) → flatten batch, time - # (B, S, T) → (S, B*T) - n_channels = value.shape[1] - value_flat = value.permute(1, 0, 2).reshape(n_channels, -1) - - batch_mean = value_flat.mean(dim=1) - batch_min = value_flat.min(dim=1).values - batch_max = value_flat.max(dim=1).values - n_samples = value_flat.shape[1] - - batch_var = value_flat.var(dim=1, unbiased=False) - batch_M2 = batch_var * n_samples - - else: - # Video (batch, time, height, width) → global statistics - value_flat = value.flatten() - - batch_mean = torch.tensor([value_flat.mean()], dtype=torch.float64) - batch_min = torch.tensor([value_flat.min()], dtype=torch.float64) - batch_max = torch.tensor([value_flat.max()], dtype=torch.float64) - n_samples = value_flat.shape[0] - - batch_var = value_flat.var(unbiased=False) - batch_M2 = batch_var * n_samples - - # Parallel Welford's algorithm for combining batches - # https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm - n_old = self.n - n_new = n_samples - n_total = n_old + n_new - - # Update mean - delta = batch_mean - self.mean - self.mean = (n_old * self.mean + n_new * batch_mean) / n_total - - # Update M2 (sum of squared deviations) - # M2_total = M2_old + M2_new + delta^2 * n_old * n_new / n_total - self.M2 = self.M2 + batch_M2 + delta * delta * n_old * n_new / n_total - - self.n = n_total - - # Update min/max - self.min_val = torch.minimum(self.min_val, batch_min) - self.max_val = torch.maximum(self.max_val, batch_max) - - def _compute_std(self): - """ - Derive sample standard deviation from the Welford M2 accumulator. - - Uses Bessel's correction (``n - 1``) when more than one sample has - been seen; falls back to zeros when ``n <= 1`` to avoid division by - zero. The result is written to :attr:`std` in-place. - - Returns - ------- - None - """ - if self.n > 1: - self.std = torch.sqrt(self.M2 / (self.n - 1)) - else: - self.std = torch.zeros_like(self.mean) - - def compute(self): - """ - Finalise and return all accumulated statistics as NumPy arrays. - - Calls :meth:`_compute_std` internally to derive the standard - deviation from the Welford M2 accumulator before returning. - Returns ``None`` if :meth:`update` was never called. - - Returns - ------- - dict or None - ``None`` if no data was ever seen. Otherwise a dictionary - with the following keys, each mapping to a - ``numpy.ndarray`` of shape ``(C,)``: - - ``'mean'`` - Per-channel arithmetic mean. - ``'std'`` - Per-channel sample standard deviation (Bessel-corrected). - ``'min_val'`` - Per-channel minimum value seen across all batches. - ``'max_val'`` - Per-channel maximum value seen across all batches. - """ - if not self.initialized: - return None - - self._compute_std() - - return { - "mean": self.mean.numpy(), - "std": self.std.numpy(), - "min_val": self.min_val.numpy(), - "max_val": self.max_val.numpy(), - } - - -def compute_preprocessing_stats( - datasets: "list[TokamakH5Dataset]", - output_path: str | Path = "preprocessing_stats.pt", - batch_size: int = 1, -) -> dict[str, dict[str, np.ndarray]]: - """ - Compute per-modality preprocessing statistics over a collection of - datasets. - - Iterates over all chunks in every dataset, accumulates running statistics - with :class:`WelfordTensor`, and saves the result to *output_path* via - :func:`torch.save`. Only modalities that appear in the loaded batches - are included in the output. - - Parameters - ---------- - datasets : list of TokamakH5Dataset - One or more dataset instances whose data will be concatenated. - Signal and movie configurations are read from ``datasets[0]``. - output_path : str or Path, optional - Filesystem path for the saved ``.pt`` statistics file. - Default is ``"preprocessing_stats.pt"``. - batch_size : int, optional - Batch size for the internal DataLoader. Default is ``1``. - - Returns - ------- - dict[str, dict[str, numpy.ndarray]] - Nested dictionary ``{modality_name: stats}``, where *stats* is the - dictionary returned by :meth:`WelfordTensor.compute`: - - ``'mean'`` - Per-channel arithmetic mean, shape ``(C,)``. - ``'std'`` - Per-channel sample standard deviation, shape ``(C,)``. - ``'min_val'`` - Per-channel minimum, shape ``(C,)``. - ``'max_val'`` - Per-channel maximum, shape ``(C,)``. - """ - from tqdm import tqdm - - # Use instance-level configs (deep copies that may have been modified). - signal_configs = datasets[0].signal_configs - movie_configs = datasets[0].movie_configs - - welford_stats = { - cfg.name: WelfordTensor() - for cfg in signal_configs + movie_configs} - - # Iterate one dataset at a time and close each file handle after use. - # Using ConcatDataset + persistent_workers causes all HDF5 file handles - # (each with a 16 MB chunk cache) to accumulate in the worker process, - # exhausting memory after ~1000 files. - for dataset in tqdm(datasets, desc="Files"): - dataloader = DataLoader( - dataset, batch_size=batch_size, collate_fn=collate_fn, - num_workers=0) - for batch in dataloader: - for modality_name, tensor in batch.items(): - if modality_name not in welford_stats: - continue - # Movies arrive as (B, C, T, H, W); flatten spatial/temporal dims - # to (B, C, T*H*W) so WelfordTensor computes per-channel stats. - if tensor.ndim == 5: - B, C, T, H, W = tensor.shape - tensor = tensor.reshape(B, C, T * H * W) - welford_stats[modality_name].update(tensor) - # Explicitly close the HDF5 file handle to free memory before next file. - if dataset.h5_file is not None: - dataset.h5_file.close() - dataset.h5_file = None - - # Only include trackers that received data - final_stats = { - modality: tracker.compute() - for modality, tracker in welford_stats.items() - if tracker.initialized - } - torch.save(final_stats, output_path) - - print(f"Saved statistics to {output_path}") - return final_stats @dataclass @@ -469,7 +104,7 @@ class SignalConfig: target_fs: float apply_stft: bool channels_to_use: Optional[slice] = None - preprocess: PreprocessConfig = None + preprocess: PreprocessConfig | None = None def __post_init__(self): if self.preprocess is None: @@ -514,7 +149,7 @@ class MovieConfig: target_fps: int # Target frames per second after resampling height: int # Frame height width: int # Frame width - preprocess: PreprocessConfig = None # Add preprocessing config + preprocess: PreprocessConfig | None = None def __post_init__(self): if self.preprocess is None: @@ -549,7 +184,7 @@ class TokamakH5Dataset(Dataset): Parameters ---------- - hdf5_path : str + hdf5_path : str | Path Path to a preprocessed HDF5 shot file (output of the data-preparation pipeline). chunk_duration_s : float, optional @@ -720,6 +355,7 @@ class TokamakH5Dataset(Dataset): ["filterscopes"], 104, 10e3, + channels_to_use=slice(0, 8), # Use only the first 8 channels apply_stft=False, preprocess=PreprocessConfig(method="log"), ), @@ -1011,7 +647,6 @@ def _update_preprocessing_stats(self): if "max_val" in stats: config.preprocess.max_val = stats["max_val"] - @profile def _apply_preprocessing( self, tensor: torch.Tensor, @@ -1046,6 +681,7 @@ def _apply_preprocessing( # Reshape per-channel statistics for correct broadcasting. # Stats have shape (C,); we add trailing singleton dims to match ndim. + reshape_dims: tuple[int, ...] | None if tensor.ndim == 4: # (C, T, H, W) — video reshape_dims = (tensor.shape[0], 1, 1, 1) @@ -1060,7 +696,8 @@ def _apply_preprocessing( if config.method == "standardize": if config.mean is None or config.std is None: - print("Warning: standardize requested but no statistics provided") + print("Warning: " + "standardize requested but no statistics provided") return tensor # Convert to tensor and reshape for broadcasting @@ -1077,7 +714,8 @@ def _apply_preprocessing( elif config.method == "normalize": if config.min_val is None or config.max_val is None: - print("Warning: normalize requested but no statistics provided") + print("Warning: " + "normalize requested but no statistics provided") return tensor min_val = torch.tensor( @@ -1092,13 +730,15 @@ def _apply_preprocessing( elif config.method == "log_standardize": # log10(x+1) in-place via numpy (2x faster than torch on CPU). - # tensor.numpy() is zero-copy; modifying arr updates tensor in-place. + # tensor.numpy() is zero-copy; + # modifying arr updates tensor in-place. arr = tensor.numpy() arr += 1 np.log10(arr, out=arr) if config.mean is None or config.std is None: - print("Warning: log_standardize requested but no statistics provided") + print("Warning: " + "log_standardize requested but no statistics provided") return tensor # Convert to tensor and reshape for broadcasting @@ -1115,6 +755,7 @@ def _apply_preprocessing( elif config.method == "log": arr = tensor.numpy() + arr = np.clip(arr, a_min=0., a_max=None, out=arr) arr += 1 np.log10(arr, out=arr) return tensor @@ -1136,7 +777,6 @@ def _open_hdf5(self): if self.h5_file is None: self.h5_file = h5py.File(self.hdf5_path, "r") - @profile def _load_signal_raw( self, f: h5py.File, @@ -1179,8 +819,14 @@ def _load_signal_raw( continue if data_group is None: + if config.channels_to_use: + num_channels = len( + range(*config.channels_to_use.indices(config.num_channels)) + ) + else: + num_channels = config.num_channels return torch.zeros( - (config.num_channels, round(duration_s * config.target_fs)) + (num_channels, round(duration_s * config.target_fs)) ) ydata_ds = data_group["ydata"] @@ -1193,17 +839,29 @@ def _load_signal_raw( n_samples = xdata_ds.shape[0] if n_samples < 2 or xdata_end_s == xdata_start_s: + if config.channels_to_use: + num_channels = len( + range(*config.channels_to_use.indices(config.num_channels)) + ) + else: + num_channels = config.num_channels return torch.zeros( - (config.num_channels, round(duration_s * config.target_fs)) + (num_channels, round(duration_s * config.target_fs)) ) # Compute actual sampling frequency from the data actual_fs = (n_samples - 1) / (xdata_end_s - xdata_start_s) # Step 1: Initialize output array (C, T) — matches HDF5 storage layout, - # avoiding a transpose and keeping all copies between contiguous arrays. + # avoiding a transpose and keeping all copies between contiguous arrays + if config.channels_to_use: + num_channels = len( + range(*config.channels_to_use.indices(config.num_channels)) + ) + else: + num_channels = config.num_channels output = np.zeros( - (config.num_channels, round(duration_s * actual_fs)), + (num_channels, round(duration_s * actual_fs)), dtype=np.float32 ) @@ -1229,8 +887,10 @@ def _load_signal_raw( data = ydata_ds[ch_slice, hdf5_start_clamped:hdf5_end_clamped] # Step 4: Calculate where to insert in output array - # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fs - # This corresponds to output index: (that_time - t_start) * actual_fs + # The loaded data starts at time: + # xdata_start_s + hdf5_start_clamped / actual_fs + # This corresponds to output index: + # (that_time - t_start) * actual_fs output_start = hdf5_start_clamped - hdf5_start output_end = output_start + data.shape[1] @@ -1294,8 +954,8 @@ def _compute_stft(self, signal: torch.Tensor) -> torch.Tensor: window=self.stft_window, return_complex=True, ) - spec = spec[:, 1:, :] # Remove DC component (extreme values) - return torch.abs(spec) + # spec = spec[:, 1:, :] # Remove DC component (extreme values) + return torch.abs(spec)[:, 1:, :] # Remove DC component (extreme value) def _load_metadata(self, f: h5py.File) -> dict: """ @@ -1353,7 +1013,6 @@ def __setstate__(self, state): """Restore state after unpickling.""" self.__dict__.update(state) - @profile def _process_signal( self, data: torch.Tensor, @@ -1390,7 +1049,6 @@ def _process_signal( processed = self._apply_preprocessing(processed, config.preprocess) return processed - @profile def _load_movie_raw( self, f: h5py.File, @@ -1477,7 +1135,11 @@ def _load_movie_raw( # Step 1: Initialize output array with zeros at actual fps # (T, C, H, W) output = np.zeros( - (raw_channels, round(duration_s * actual_fps), raw_height, raw_width), + ( + raw_channels, round(duration_s * actual_fps), + raw_height, + raw_width + ), dtype=np.float32 ) @@ -1497,8 +1159,10 @@ def _load_movie_raw( data[np.isnan(data)] = 0 # Step 4: Calculate where to insert in output array - # The loaded data starts at time: xdata_start_s + hdf5_start_clamped / actual_fps - # This corresponds to output index: (that_time - t_start) * actual_fps + # The loaded data starts at time: + # xdata_start_s + hdf5_start_clamped / actual_fps + # This corresponds to output index: + # (that_time - t_start) * actual_fps output_start = hdf5_start_clamped - hdf5_start output_end = output_start + data.shape[1] @@ -1520,11 +1184,15 @@ def _load_movie_raw( # Step 5: Convert to tensor and resample to target fps and dimensions tensor = torch.from_numpy(output) - # Resample using trilinear interpolation within each channel independently. + # Resample using trilinear interpolation within channels independently. # F.interpolate treats dim-1 as channels (not interpolated across); # the 3D kernel blends only within each channel's (T, H, W) volume. # (C, T, H, W) → (1, C, T, H, W) → trilinear → (C, T', H', W') - target_size = (round(duration_s * config.target_fps), config.height, config.width) + target_size = ( + round(duration_s * config.target_fps), + config.height, + config.width + ) if tensor.shape[1:] != torch.Size(target_size): tensor = F.interpolate( tensor.unsqueeze(0), @@ -1562,7 +1230,6 @@ def __getitem__(self, idx: int) -> dict: else: return self._getitem_standard(idx) - @profile def _getitem_standard(self, idx: int) -> dict: """ Load and return the data chunk at *idx* in standard mode. @@ -1591,8 +1258,14 @@ def _getitem_standard(self, idx: int) -> dict: all_signals = {} for config in self.signal_configs: if config.name in self.input_signals: - raw_data = self._load_signal_raw(self.h5_file, config, t_start, t_end) - all_signals[config.name] = self._process_signal(raw_data, config) + raw_data = self._load_signal_raw( + self.h5_file, + config, t_start, + t_end + ) + all_signals[config.name] = self._process_signal( + raw_data, config + ) # Load and process movies all_movies = {} @@ -1646,7 +1319,9 @@ def _getitem_prediction(self, idx: int) -> dict: for config in self.signal_configs: if config.name not in signals_to_load: continue - raw_data = self._load_signal_raw(self.h5_file, config, t_start, t_end) + raw_data = self._load_signal_raw( + self.h5_file, config, t_start, t_end + ) all_signals[config.name] = self._process_signal(raw_data, config) # Load and process movies @@ -1654,9 +1329,12 @@ def _getitem_prediction(self, idx: int) -> dict: for movie_config in self.movie_configs: if movie_config.name not in signals_to_load: continue - raw_movie = self._load_movie_raw(self.h5_file, movie_config, t_start, t_end) + raw_movie = self._load_movie_raw( + self.h5_file, movie_config, t_start, t_end + ) all_movies[movie_config.name] = self._apply_preprocessing( - raw_movie, movie_config.preprocess) + raw_movie, movie_config.preprocess + ) # Load metadata all_metadata = self._load_metadata(self.h5_file) @@ -1676,7 +1354,9 @@ def _getitem_prediction(self, idx: int) -> dict: self.chunk_duration_s * config.target_fs / self.hop_length ) else: - n_training_frames = round(self.chunk_duration_s * config.target_fs) + n_training_frames = round( + self.chunk_duration_s * config.target_fs + ) if config.name in self.input_signals: inputs[config.name] = signal[..., :n_training_frames] @@ -1690,7 +1370,9 @@ def _getitem_prediction(self, idx: int) -> dict: continue movie_name = movie_config.name movie_data = all_movies[movie_name] - n_training_frames = round(self.chunk_duration_s * movie_config.target_fps) + n_training_frames = round( + self.chunk_duration_s * movie_config.target_fps + ) # movie_data shape: (C, extended_movie_frames, height, width) if movie_name in self.input_signals: inputs[movie_name] = movie_data[:, :n_training_frames] diff --git a/src/tokamak_foundation_model/data/multi_file_dataset.py b/src/tokamak_foundation_model/data/multi_file_dataset.py index dd6029a..3ca4276 100644 --- a/src/tokamak_foundation_model/data/multi_file_dataset.py +++ b/src/tokamak_foundation_model/data/multi_file_dataset.py @@ -286,6 +286,10 @@ def _get_file_handle(self, file_idx: int) -> h5py.File: # Dataset interface # ------------------------------------------------------------------------- + def _open_hdf5(self) -> None: + """No-op: file handles are opened on demand via the LRU cache.""" + pass + def __len__(self) -> int: return int(self._cumulative_lengths[-1]) diff --git a/src/tokamak_foundation_model/data/preprocess_data.py b/src/tokamak_foundation_model/data/preprocess_data.py index 9e42831..650a68c 100644 --- a/src/tokamak_foundation_model/data/preprocess_data.py +++ b/src/tokamak_foundation_model/data/preprocess_data.py @@ -2,7 +2,7 @@ import numpy as np from pathlib import Path from typing import Optional -from torch.utils.data import DataLoader, SubsetRandomSampler +from torch.utils.data import DataLoader, SubsetRandomSampler, SequentialSampler from .multi_file_dataset import TokamakMultiFileDataset from .data_loader import collate_fn, collate_fn_prediction @@ -356,7 +356,7 @@ def compute_preprocessing_stats( dataloader = DataLoader( dataset, batch_size=batch_size, - sampler=SubsetRandomSampler(indices), + sampler=SequentialSampler(indices), num_workers=num_workers, collate_fn=collate, pin_memory=False, diff --git a/src/tokamak_foundation_model/models/model_factory.py b/src/tokamak_foundation_model/models/model_factory.py index c30f8f4..23bc26f 100644 --- a/src/tokamak_foundation_model/models/model_factory.py +++ b/src/tokamak_foundation_model/models/model_factory.py @@ -17,7 +17,7 @@ "ech": "actuator", "pin": "actuator", "tin": "actuator", - "d_alpha": "fast_time_series", + "filterscopes": "fast_time_series", "mse": "profile", "ts_core_density": "profile", "mhr": "spectrogram", @@ -35,7 +35,6 @@ "profile": SpatialProfileBaselineAutoEncoder, "spectrogram": SpectrogramBaselineAutoEncoder, "spectrogram_tf_attn": SpectrogramTFAttnAutoEncoder, - "spectrogram_res_lstm": SpectrogramResLSTMAutoEncoder, "video": VideoBaselineAutoEncoder, } diff --git a/src/tokamak_foundation_model/trainer/trainer.py b/src/tokamak_foundation_model/trainer/trainer.py index fdca48c..109f0bc 100644 --- a/src/tokamak_foundation_model/trainer/trainer.py +++ b/src/tokamak_foundation_model/trainer/trainer.py @@ -14,6 +14,7 @@ logger = logging.getLogger(__name__) + class MultimodalTrainer: def __init__( self, @@ -34,11 +35,16 @@ def __init__( def _train_epoch(self, dataloader: DataLoader): self.model.train() total_loss = 0 + n_batches = len(dataloader) # type: ignore[arg-type] for batch_idx, batch in enumerate(dataloader): inputs = batch['inputs'] targets = batch['targets'] - inputs = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()} - targets = {k: v.to(self.device) if isinstance(v, torch.Tensor) else v for k, v in targets.items()} + inputs = { + k: v.to(self.device) if isinstance(v, torch.Tensor) + else v for k, v in inputs.items()} + targets = { + k: v.to(self.device) if isinstance(v, torch.Tensor) + else v for k, v in targets.items()} self.optimizer.zero_grad() outputs = self.model(inputs) @@ -48,15 +54,15 @@ def _train_epoch(self, dataloader: DataLoader): total_loss += loss.item() if batch_idx % 10 == 0: - print(f" Batch {batch_idx}/{len(dataloader)}," - f" Loss: {loss.item():.4f}") - return total_loss / len(dataloader) + print(f" Batch {batch_idx}/{n_batches}, Loss: {loss.item():.4f}") + return total_loss / n_batches - def _validate_epoch(self, dataloader: DataLoader): + def _validate_epoch(self, dataloader: DataLoader) -> float: self.model.eval() total_loss = 0 + n_batches = len(dataloader) # type: ignore[arg-type] with torch.no_grad(): - for batch_idx, batch in enumerate(dataloader): + for batch in dataloader: inputs = batch["inputs"] targets = batch["targets"] inputs = { @@ -71,12 +77,12 @@ def _validate_epoch(self, dataloader: DataLoader): outputs = self.model(inputs) loss = self.loss_fn(outputs, targets) total_loss += loss.item() - return total_loss / len(dataloader) + return total_loss / n_batches def train( self, train_dataloader: DataLoader, - val_dataloader: DataLoader = None + val_dataloader: DataLoader | None = None ): best_val_loss = float("inf") for epoch in range(self.epochs): @@ -107,19 +113,20 @@ def load_checkpoint(self, checkpoint_path=None): class UnimodalTrainer: - def __init__(self, - epochs: int, - model: nn.Module, - loss_fn: nn.Module, - optimizer: optim.Optimizer, - scheduler: optim.lr_scheduler.LRScheduler | None = None, - distributed_manager: DistributedManager | None = None, - tracker: Tracker | None = None, - drawer: DrawerProtocol | None = None, - metrics: list[Metric] | None = None, - checkpoint_path: str | Path = "checkpoint.pth", - log_interval: int = 1, - ): + def __init__( + self, + epochs: int, + model: nn.Module, + loss_fn: nn.Module, + optimizer: optim.Optimizer, + scheduler: optim.lr_scheduler.LRScheduler | None = None, + distributed_manager: DistributedManager | None = None, + tracker: Tracker | None = None, + drawer: DrawerProtocol | None = None, + metrics: list[Metric] | None = None, + checkpoint_path: str | Path = "checkpoint.pth", + log_interval: int = 1, + ): self.epochs = epochs self.log_interval = log_interval @@ -141,11 +148,14 @@ def __init__(self, self.metrics: list[Metric] = metrics if metrics else [] # Paths - self.checkpoint_path = Path(checkpoint_path) if checkpoint_path else None - self.best_checkpoint_path = self.checkpoint_path.with_name( # type: ignore - self.checkpoint_path.stem + "_best" + self.checkpoint_path.suffix # type: ignore + self.checkpoint_path: Path | None = ( + Path(checkpoint_path) if checkpoint_path else None + ) + self.best_checkpoint_path: Path | None = ( + self.checkpoint_path.with_name( + self.checkpoint_path.stem + "_best" + self.checkpoint_path.suffix + ) if self.checkpoint_path else None ) - def _train_step(self, batch: dict): data = batch[self.modality_key].to(self.dm.device) @@ -187,11 +197,13 @@ def _validate_epoch(self, dataloader: DataLoader): def _log_train(self, epoch: int): train_mean = self.tracker.metrics["train"]["mean"]["loss"]() - logger.info(f"Epoch {epoch+1}/{self.epochs}, Train Loss: {train_mean:.4f}") + logger.info( + f"Epoch {epoch + 1}/{self.epochs}, Train Loss: {train_mean:.4f}" + ) def _log_validate(self, epoch: int): val_mean = self.tracker.metrics["validate"]["mean"]["loss"]() - text = [f"Epoch {epoch+1}/{self.epochs}, Val Loss: {val_mean:.4f}"] + text = [f"Epoch {epoch + 1}/{self.epochs}, Val Loss: {val_mean:.4f}"] for key in self.tracker.metrics["validate"]["value"]: if key != "loss": val = self.tracker.metrics["validate"]["mean"][key]() @@ -199,12 +211,12 @@ def _log_validate(self, epoch: int): logger.info(", ".join(text)) def _save_checkpoint(self, epoch: int): - if not self.dm.is_main: + if not self.dm.is_main or self.checkpoint_path is None: return raw_model = self.dm.unwrap(self.model) torch.save( { - "model_state_dict": raw_model.state_dict(), # type: ignore + "model_state_dict": raw_model.state_dict(), # type: ignore[union-attr] "optimizer_state_dict": self.optimizer.state_dict(), "scheduler_state_dict": ( self.scheduler.state_dict() if self.scheduler else None @@ -212,23 +224,24 @@ def _save_checkpoint(self, epoch: int): "tracker_state_dict": self.tracker.state_dict(), "epoch": epoch, }, - self.checkpoint_path, # type: ignore + self.checkpoint_path, ) def _save_best(self): - if not self.dm.is_main: + if not self.dm.is_main or self.best_checkpoint_path is None: return if self.tracker.is_best("validate", "loss"): raw_model = self.dm.unwrap(self.model) - torch.save(raw_model.state_dict(), self.best_checkpoint_path) # type: ignore + torch.save(raw_model.state_dict(), self.best_checkpoint_path) logger.info("Best model checkpoint saved!") - def fit(self, - train_dataloader: DataLoader, - val_dataloader: DataLoader | None = None, - modality_key: str | None = None, - train_sampler=None, - ): + def fit( + self, + train_dataloader: DataLoader, + val_dataloader: DataLoader | None = None, + modality_key: str | None = None, + train_sampler=None, + ): if modality_key is None: raise ValueError("modality_key is required for unimodal training") self.modality_key = modality_key @@ -240,15 +253,19 @@ def fit(self, for metric in self.metrics: metric.to(self.dm.device) - n_train = len(train_dataloader) + n_train = len(train_dataloader) # type: ignore[arg-type] # Set up tracking - self._train_step = self.tracker.track("train", n_train)(self._train_step) - self._log_train = self.tracker.log("train", "mean")(self._log_train) + track_train = self.tracker.track("train", n_train) + self._train_step = track_train(self._train_step) # type: ignore + log_train = self.tracker.log("train", "mean") + self._log_train = log_train(self._log_train) # type: ignore if val_dataloader is not None: - n_val = len(val_dataloader) - self._validate_step = self.tracker.track("validate", n_val)(self._validate_step) - self._log_validate = self.tracker.log("validate", "mean")(self._log_validate) + n_val = len(val_dataloader) # type: ignore[arg-type] + track_val = self.tracker.track("validate", n_val) + self._validate_step = track_val(self._validate_step) # type: ignore + log_val = self.tracker.log("validate", "mean") + self._log_validate = log_val(self._log_validate) # type: ignore drawing_path = self.checkpoint_path.parent / "plots" # type: ignore self.drawer.setup(train_dataloader, drawing_path, modality_key) @@ -270,7 +287,9 @@ def fit(self, self.dm.barrier() if (epoch + 1) % self.log_interval == 0 and self.dm.is_main: - val_loss = self.tracker.metrics["validate"]["mean"]["loss"]() if val_dataloader is not None else None + val_loss = ( + self.tracker.metrics["validate"]["mean"]["loss"]()) \ + if val_dataloader is not None else None train_loss = self.tracker.metrics["train"]["mean"]["loss"]() self.drawer( model=self.dm.unwrap(self.model), # type: ignore @@ -297,12 +316,16 @@ def load_checkpoint(self, checkpoint_path=None): if path is None or not os.path.exists(path): logger.info(f"No checkpoint found at: {path}") return - checkpoint = torch.load(path, map_location=self.dm.device) + checkpoint = torch.load( + path, map_location=self.dm.device, weights_only=False + ) raw_model = self.dm.unwrap(self.model) - raw_model.load_state_dict(checkpoint["model_state_dict"]) # type: ignore + raw_model.load_state_dict(checkpoint["model_state_dict"]) self.optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) if self.scheduler and checkpoint.get("scheduler_state_dict"): self.scheduler.load_state_dict(checkpoint["scheduler_state_dict"]) if checkpoint.get("tracker_state_dict"): self.tracker.load_state_dict(checkpoint["tracker_state_dict"]) - logger.info(f"Resumed from checkpoint: {path} (epoch {checkpoint.get('epoch', '?')})") \ No newline at end of file + logger.info( + f"Resumed from checkpoint: {path} " + f"(epoch {checkpoint.get('epoch', '?')})") diff --git a/src/tokamak_foundation_model/utils/drawing.py b/src/tokamak_foundation_model/utils/drawing.py index 44720d7..b5125b6 100644 --- a/src/tokamak_foundation_model/utils/drawing.py +++ b/src/tokamak_foundation_model/utils/drawing.py @@ -71,7 +71,7 @@ def __call__(self, model: torch.nn.Module, epoch: int, train_loss: float, val_lo output = output[0] output = output[0].cpu() - ax2.imshow(output[self.channel].numpy(), cmap='viridis', origin='lower', aspect='auto') + # ax2.imshow(output[self.channel].numpy(), cmap='viridis', origin='lower', aspect='auto') ax2.set_axis_off() val_str = f" | Val L1={val_loss:.6f}" if val_loss is not None else "" diff --git a/src/tokamak_foundation_model/utils/tracking.py b/src/tokamak_foundation_model/utils/tracking.py index 40673fb..f277ed8 100644 --- a/src/tokamak_foundation_model/utils/tracking.py +++ b/src/tokamak_foundation_model/utils/tracking.py @@ -23,17 +23,18 @@ def default_list(): class Mean: """Keeps track of the running mean, along with the latest value.""" - def __init__(self): - self.reset() + def __init__(self) -> None: + self.count: int = 0 + self.total: float = 0. def __call__(self): return self.total / max(self.count, 1) - def reset(self): + def reset(self) -> None: self.count = 0 - self.total = 0 + self.total = 0. - def update(self, val): + def update(self, val: float) -> None: if math.isfinite(val): self.count += 1 self.total += val @@ -74,12 +75,12 @@ def __init__( rank: int = 0, step: int = 0, ): - self.metrics = {} - self.history = {} + self.metrics: dict = {} + self.history: dict = {} self.writer = writer self.rank = rank self.step = step - self._progress = {} # label -> {completed, total} + self._progress: dict = {} # label -> {completed, total} def _write(self, msg: str): print(msg) @@ -116,13 +117,13 @@ def done(self, label: str, title: str): self._write(f" {k}: {m():.6f}") def track( - self, - label: str, - length: int, - completed: int = 0, - log_every: int = 0, - op: dist.ReduceOp = dist.ReduceOp.AVG, - ddp_active: bool = "LOCAL_RANK" in os.environ, + self, + label: str, + length: int, + completed: int = 0, + log_every: int = 0, + op: dist.ReduceOp.RedOpType = dist.ReduceOp.AVG, + ddp_active: bool = "LOCAL_RANK" in os.environ, ): self._progress[label] = {"completed": completed, "total": length} self.metrics[label] = { From a4e6e9d314b31c8c837f7280c0cf000c157b8961 Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 10 Mar 2026 11:04:00 -0400 Subject: [PATCH 28/30] drawing.py: - PEP-8 corrections - Support plots of time signals and videos Train-val-test split in fast_time_series_reconstruction.py --- .../fast_time_series_reconstruction.py | 57 +++- src/tokamak_foundation_model/utils/drawing.py | 273 ++++++++++++++++-- 2 files changed, 294 insertions(+), 36 deletions(-) diff --git a/scripts/training/fast_time_series_reconstruction.py b/scripts/training/fast_time_series_reconstruction.py index c58190b..b15467b 100644 --- a/scripts/training/fast_time_series_reconstruction.py +++ b/scripts/training/fast_time_series_reconstruction.py @@ -2,6 +2,7 @@ import argparse import logging +import random import torch import torch.nn as nn import torch.optim as optim @@ -108,7 +109,7 @@ def main(): "--log_interval", type=int, default=1, help="Plot every N epochs" ) parser.add_argument( - "--resume", action="store_true", default=False, + "--resume", action="store_true", default=True, help="Resume training from checkpoint" ) args = parser.parse_args() @@ -127,21 +128,45 @@ def main(): ### Dataset Setup ### hdf5_files = sorted(data_dir.glob("*_processed.h5")) + random.seed(42) + n = len(hdf5_files) + n_val = int(.1 * n) + n_test = int(.1 * n) + + train_paths = hdf5_files[n_val + n_test:] + val_paths = hdf5_files[:n_val] + test_paths = hdf5_files[n_val:n_val + n_test] + stats = torch.load(statistics_path, weights_only=False) - dataset_processed = TokamakMultiFileDataset( - hdf5_paths=hdf5_files, + shared_kwargs = dict( + preprocessing_stats=stats, input_signals=[signal_name], target_signals=[signal_name], n_fft=args.n_fft, hop_length=args.hop_length, - preprocessing_stats=stats, prediction_mode=False, - lengths_cache_path="../slurm/dataset_lengths.pt", ) + train_dataset = TokamakMultiFileDataset( + train_paths, + lengths_cache_path="lengths_train.pt", + **shared_kwargs + ) + validation_dataset = TokamakMultiFileDataset( + val_paths, + lengths_cache_path="lengths_validation.pt", + **shared_kwargs + ) + test_dataset = TokamakMultiFileDataset( + test_paths, + lengths_cache_path="lengths_test.pt", + **shared_kwargs + ) + + # Not sure if this is elegant - sample_data = next(iter(dataset_processed))[signal_name] + sample_data = next(iter(train_dataset))[signal_name] n_channels = sample_data.shape[0] logger.info(f"Sample data shape: {sample_data.shape}, n_channels: {n_channels}") @@ -165,8 +190,17 @@ def main(): loss_fn = nn.L1Loss() - dataloader = make_dataloader( - dataset_processed, + train_dataloader = make_dataloader( + train_dataset, + batch_size=args.batch_size, + num_workers=args.num_workers, + shuffle=True, + pin_memory=True, + prefetch_factor=args.prefetch_factor, + ) + + validation_dataloader = make_dataloader( + validation_dataset, batch_size=args.batch_size, num_workers=args.num_workers, shuffle=True, @@ -183,7 +217,7 @@ def main(): optimizer=optimizer, scheduler=lr_scheduler, checkpoint_path=checkpoint_path, - drawer=None, # drawer, + drawer=drawer, log_interval=args.log_interval, ) @@ -191,7 +225,10 @@ def main(): logger.info(f"Resuming training from checkpoint: {checkpoint_path}") trainer.load_checkpoint(checkpoint_path=checkpoint_path) - trainer.fit(dataloader, modality_key=signal_name) + trainer.fit( + train_dataloader, + validation_dataloader, + modality_key=signal_name) if __name__ == "__main__": diff --git a/src/tokamak_foundation_model/utils/drawing.py b/src/tokamak_foundation_model/utils/drawing.py index b5125b6..75b3ca7 100644 --- a/src/tokamak_foundation_model/utils/drawing.py +++ b/src/tokamak_foundation_model/utils/drawing.py @@ -1,41 +1,141 @@ +from collections.abc import Sized from pathlib import Path -from typing import Protocol, runtime_checkable +from typing import Optional, Protocol, runtime_checkable -import numpy as np import matplotlib.pyplot as plt +import numpy as np import torch from torch.utils.data import DataLoader @runtime_checkable class DrawerProtocol(Protocol): - def setup(self, dataloader: DataLoader, drawing_path: Path, modality_key: str) -> None: ... - def __call__(self, model: torch.nn.Module, epoch: int, train_loss: float, val_loss: float | None = None) -> None: ... + """ + Protocol for training-progress visualization callbacks. + + Implementors must provide :meth:`setup` and :meth:`__call__` with the + signatures below. :class:`NullDrawer` and :class:`DefaultDrawer` are + the two built-in implementations. + """ + + def setup( + self, + dataloader: DataLoader, + drawing_path: Path, + modality_key: str, + ): + ... + + def __call__( + self, + model: torch.nn.Module, + epoch: int, + train_loss: float, + val_loss: Optional[float] = None, + ): + ... class NullDrawer: """No-op drawer for non-main processes or when visualization is disabled.""" - def setup(self, dataloader: DataLoader, drawing_path: Path, modality_key: str) -> None: + def setup( + self, + dataloader: DataLoader, + drawing_path: Path, + modality_key: str, + ): pass - def __call__(self, model: torch.nn.Module, epoch: int, train_loss: float, val_loss: float | None = None) -> None: + def __call__( + self, + model: torch.nn.Module, + epoch: int, + train_loss: float, + val_loss: Optional[float] = None, + ): pass class DefaultDrawer: + """ + Visualizes training progress after each epoch. + + Saves two persistent plots to *drawing_path* (overwritten each epoch): + + * ``loss_curve.png`` — cumulative train and optional validation loss over + epochs. + * ``reconstruction.png`` — input vs. model output for a fixed probe + sample. The visualization adapts to the channel dimensionality: + + ========= =========================== =============================== + ``ndim`` Interpretation Plot type + ========= =========================== =============================== + 3 ``(T, H, W)`` — video Uniform strip of frames + 2 ``(H, W)`` — spectrogram :func:`~matplotlib.pyplot.imshow` + 1 ``(T,)`` — signal :func:`~matplotlib.pyplot.plot` + ========= =========================== =============================== - def __init__(self, plot_channel: int | None = None): - self._plot_channel: int | None = plot_channel + Parameters + ---------- + plot_channel : int or None, optional + Index of the channel to visualize. If ``None`` (default), the + middle channel (``C // 2``) is selected automatically. - def setup(self, dataloader: DataLoader, drawing_path: Path, modality_key: str) -> None: + Attributes + ---------- + drawing_path : Path + Directory where plots are saved. Set by :meth:`setup`. + probe_sample : torch.Tensor + Fixed sample used for reconstruction plots. Shape ``(C, ...)``. + Set by :meth:`setup`. + channel : int + Channel index used for visualization. Set by :meth:`setup`. + train_losses : list of float + Accumulated training losses, one entry per :meth:`__call__`. + val_losses : list of float + Accumulated validation losses. Only populated when *val_loss* is + passed to :meth:`__call__`. + """ + + _NUM_VIDEO_FRAMES = 6 # number of frames shown in the video strip + + def __init__( + self, + plot_channel: Optional[int] = None, + ): + self._plot_channel: Optional[int] = plot_channel + + def setup( + self, + dataloader: DataLoader, + drawing_path: Path, + modality_key: str, + ): + """Initialize the drawer with dataset and output directory. + + Must be called once before the first :meth:`__call__`. Selects a + fixed probe sample from the dataset and creates *drawing_path*. + + Parameters + ---------- + dataloader : DataLoader + Training dataloader. Its ``dataset`` attribute is used to + retrieve the probe sample. + drawing_path : Path + Directory where ``loss_curve.png`` and ``reconstruction.png`` + will be written. Created if it does not exist. + modality_key : str + Key used to index into each dataset sample dict (e.g. + ``'spectrogram'``). + """ self.drawing_path = Path(drawing_path) self.drawing_path.mkdir(parents=True, exist_ok=True) self.modality_key = modality_key dataset = dataloader.dataset + assert isinstance(dataset, Sized), "Dataset must implement __len__" idx = min(10, len(dataset) - 1) - # idx = 30840 self.probe_sample = dataset[idx][modality_key] if self._plot_channel is not None: @@ -43,39 +143,160 @@ def setup(self, dataloader: DataLoader, drawing_path: Path, modality_key: str) - else: self.channel = self.probe_sample.shape[0] // 2 - # self.channel = 19 - self.train_losses: list[float] = [] self.val_losses: list[float] = [] @torch.no_grad() - def __call__(self, model: torch.nn.Module, epoch: int, train_loss: float, val_loss: float | None = None) -> None: + def __call__( + self, + model: torch.nn.Module, + epoch: int, + train_loss: float, + val_loss: Optional[float] = None, + ): + """Record losses and save visualization plots for the current epoch. + + Parameters + ---------- + model : torch.nn.Module + Trained model, run in eval mode to produce the reconstruction. + epoch : int + Zero-based epoch index. + train_loss : float + Training loss for this epoch. + val_loss : float or None, optional + Validation loss for this epoch, or ``None`` if no validation was + performed. Default is ``None``. + """ self.train_losses.append(train_loss) if val_loss is not None: self.val_losses.append(val_loss) - model.eval() - fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4)) + self._save_loss_curve() + self._save_reconstruction(model, epoch, train_loss, val_loss) - ax1.plot(self.train_losses, color='blue', label='Train') + def _save_loss_curve(self): + """Write ``loss_curve.png``, overwriting any previous version.""" + fig, ax = plt.subplots(figsize=(6, 4)) + ax.plot(self.train_losses, color='blue', label='Train') if self.val_losses: - ax1.plot(self.val_losses, color='orange', label='Val') - ax1.set_xlabel('Log Step') - ax1.set_ylabel('Loss') - ax1.legend() - ax1.grid(True) + ax.plot(self.val_losses, color='orange', label='Val') + ax.set_xlabel('Epoch') + ax.set_ylabel('Loss') + ax.legend() + ax.grid(True) + fig.tight_layout() + fig.savefig(self.drawing_path / "loss_curve.png") + plt.close(fig) + def _save_reconstruction( + self, + model: torch.nn.Module, + epoch: int, + train_loss: float, + val_loss: Optional[float], + ): + """Write ``reconstruction.png``, overwriting any previous version. + + Runs the probe sample through *model* and dispatches to the + appropriate helper based on the channel dimensionality (3-D video, + 2-D spectrogram, or 1-D signal). + """ + model.eval() x = self.probe_sample.unsqueeze(0).to(next(model.parameters()).device) output = model(x) if isinstance(output, tuple): output = output[0] output = output[0].cpu() - # ax2.imshow(output[self.channel].numpy(), cmap='viridis', origin='lower', aspect='auto') - ax2.set_axis_off() + input_data = self.probe_sample[self.channel].numpy() + recon_data = output[self.channel].numpy() + + title = f"Epoch {epoch + 1} | Train L1={train_loss:.6f}" + if val_loss is not None: + title += f" | Val L1={val_loss:.6f}" + + if recon_data.ndim == 3: + self._plot_video(input_data, recon_data, title) + else: + self._plot_2d_or_1d(input_data, recon_data, title) + + def _plot_video( + self, + input_data: np.ndarray, + recon_data: np.ndarray, + title: str, + ): + """ + Save a frame-strip comparison for video tensors of shape ``(T, H, W)``. + + Selects :attr:`_NUM_VIDEO_FRAMES` frames uniformly across the time + axis and lays them out in two rows (input on top, reconstruction + below). + + Parameters + ---------- + input_data : numpy.ndarray + Ground-truth video, shape ``(T, H, W)``. + recon_data : numpy.ndarray + Model reconstruction, shape ``(T, H, W)``. + title : str + Figure super-title. + """ + n = self._NUM_VIDEO_FRAMES + indices = np.linspace(0, input_data.shape[0] - 1, n, dtype=int) + + fig, axes = plt.subplots(2, n, figsize=(2 * n, 4)) + for col, t in enumerate(indices): + for row, data in enumerate((input_data, recon_data)): + axes[row, col].imshow( + data[t], cmap='viridis', origin='lower', aspect='auto', + ) + axes[row, col].set_axis_off() + axes[0, col].set_title(f't={t}', fontsize=8) - val_str = f" | Val L1={val_loss:.6f}" if val_loss is not None else "" - fig.suptitle(f"Epoch {epoch+1} | Train L1={train_loss:.6f}{val_str}") + fig.text(0.01, 0.75, 'Input', va='center', rotation='vertical', fontsize=9) + fig.text( + 0.01, 0.25, 'Reconstruction', va='center', rotation='vertical', fontsize=9, + ) + fig.suptitle(title) + fig.tight_layout(rect=(0.03, 0, 1, 1)) + fig.savefig(self.drawing_path / "reconstruction.png") + plt.close(fig) + + def _plot_2d_or_1d( + self, + input_data: np.ndarray, + recon_data: np.ndarray, + title: str, + ): + """ + Save an input/reconstruction comparison for 2-D or 1-D tensors. + + Parameters + ---------- + input_data : numpy.ndarray + Ground-truth data, shape ``(H, W)`` or ``(T,)``. + recon_data : numpy.ndarray + Model reconstruction, same shape as *input_data*. + title : str + Figure super-title. + """ + if recon_data.ndim == 2: + fig, axs = plt.subplots(1, 2, figsize=(8, 4), sharex="all", sharey="all") + axs[0].imshow(input_data, cmap='viridis', origin='lower', aspect='auto') + axs[0].set_axis_off() + axs[1].imshow(recon_data, cmap='viridis', origin='lower', aspect='auto') + axs[1].set_axis_off() + axs[0].set_title('Input') + axs[1].set_title('Reconstruction') + else: + fig, axs = plt.subplots(figsize=(8, 4)) + axs.plot(input_data, label="Input") + axs.plot(recon_data, label="Reconstruction") + axs.set_xlabel('Time') + axs.legend() + fig.suptitle(title) fig.tight_layout() - fig.savefig(self.drawing_path / f"probe_epoch_{epoch+1:03d}.png") + fig.savefig(self.drawing_path / "reconstruction.png") plt.close(fig) From 0aa3bcdc1fb00723fddde9cb73d57efd5e0b8179 Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 10 Mar 2026 11:06:44 -0400 Subject: [PATCH 29/30] Added functionalities for preprocessing. - Convert float64 to float32 in an existing H5 file - Add new data from another H5 file to an existing target file --- scripts/data_fetching_omega/add_to_h5.py | 164 ++++++ .../data_fetching_omega/rename_h5_groups.py | 495 ++++++++++++++++++ .../convert_float64_to_float32.py | 29 + 3 files changed, 688 insertions(+) create mode 100644 scripts/data_fetching_omega/add_to_h5.py create mode 100644 scripts/data_fetching_omega/rename_h5_groups.py create mode 100644 scripts/data_preparation/convert_float64_to_float32.py diff --git a/scripts/data_fetching_omega/add_to_h5.py b/scripts/data_fetching_omega/add_to_h5.py new file mode 100644 index 0000000..1a2027e --- /dev/null +++ b/scripts/data_fetching_omega/add_to_h5.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +""" +Add/overwrite data from source H5 files to target H5 file. +Existing signals are overwritten, new signals are added. + +Usage: + python add_to_h5.py target.h5 source1.h5 source2.h5 ... + python add_to_h5.py 200000.h5 200000_chiron.h5 200000_extra.h5 +""" + +import h5py +import sys +import argparse +from pathlib import Path + + +def add_to_h5(target_file, source_files, verbose=True): + """ + Add/overwrite trees/signals from source files to target file. + + Args: + target_file: Path to target HDF5 file (modified in place) + source_files: List of source HDF5 files to add from + verbose: Print progress messages + """ + if not Path(target_file).exists(): + print(f"Error: Target file does not exist: {target_file}") + print("Create it first or use one of the source files as target") + sys.exit(1) + + if verbose: + print(f"Target file: {target_file}") + print(f"Mode: Overwrite existing signals, add new ones\n") + + with h5py.File(target_file, 'a') as f_target: + stats = { + 'files_processed': 0, + 'trees_added': 0, + 'signals_added': 0, + 'signals_overwritten': 0 + } + + for source_file in source_files: + if not Path(source_file).exists(): + print(f"Warning: {source_file} does not exist, skipping") + continue + + if Path(source_file).resolve() == Path(target_file).resolve(): + if verbose: + print(f"Skipping {source_file} (same as target)") + continue + + if verbose: + print(f"Adding from: {source_file}") + + try: + with h5py.File(source_file, 'r') as f_source: + # Iterate over shots + for shot_name in f_source.keys(): + if verbose: + print(f" Shot {shot_name}:") + + # Ensure shot exists in target + if shot_name not in f_target: + f_target.create_group(shot_name) + if verbose: + print(f" Created shot group") + + # Iterate over trees + for tree_name in f_source[shot_name].keys(): + tree_path = f"{shot_name}/{tree_name}" + + if tree_path not in f_target: + f_target.create_group(tree_path) + stats['trees_added'] += 1 + if verbose: + print(f" Tree {tree_name} (new)") + else: + if verbose: + print(f" Tree {tree_name} (existing)") + + # Iterate over signals + for signal_name in f_source[shot_name][tree_name].keys(): + signal_path = f"{shot_name}/{tree_name}/{signal_name}" + + # Check if signal exists + if signal_path in f_target: + # Overwrite + del f_target[signal_path] + f_source.copy(f_source[signal_path], f_target, + signal_path) + stats['signals_overwritten'] += 1 + if verbose: + print(f" {signal_name} (overwritten)") + else: + # Add new + f_source.copy(f_source[signal_path], f_target, + signal_path) + stats['signals_added'] += 1 + if verbose: + print(f" {signal_name} (added)") + + stats['files_processed'] += 1 + + except Exception as e: + print(f"Error processing {source_file}: {e}") + import traceback + traceback.print_exc() + continue + + # Print summary + if verbose: + print("\n" + "=" * 60) + print("Summary:") + print("=" * 60) + print(f"Files processed: {stats['files_processed']}") + print(f"Trees added: {stats['trees_added']}") + print(f"Signals added: {stats['signals_added']}") + print(f"Signals overwritten: {stats['signals_overwritten']}") + print(f"\nTarget file updated: {target_file}") + + +def main(): + parser = argparse.ArgumentParser( + description='Add/overwrite data from source H5 files to target H5 file', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Add/update chiron data to existing file + python add_to_h5.py 200000.h5 200000_chiron.h5 + + # Add multiple sources (later sources overwrite earlier ones) + python add_to_h5.py 200000.h5 source1.h5 source2.h5 source3.h5 + + # Update all files in directory with new data + for file in *.h5; do + python add_to_h5.py "$file" updated_data.h5 + done + +Behavior: + - If signal exists in target: OVERWRITE with source data + - If signal is new: ADD to target + - Trees are merged (not replaced entirely) + """ + ) + + parser.add_argument('target', help='Target HDF5 file (will be modified)') + parser.add_argument('sources', nargs='+', + help='Source HDF5 files to add/overwrite from') + parser.add_argument('-q', '--quiet', action='store_true', + help='Suppress progress messages') + + args = parser.parse_args() + + # Add/overwrite data + add_to_h5( + args.target, + args.sources, + verbose=not args.quiet + ) + + +if __name__ == '__main__': + main() diff --git a/scripts/data_fetching_omega/rename_h5_groups.py b/scripts/data_fetching_omega/rename_h5_groups.py new file mode 100644 index 0000000..250fcf5 --- /dev/null +++ b/scripts/data_fetching_omega/rename_h5_groups.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +""" +Rename groups in HDF5 files. +If new name exists, checks if data is identical before proceeding. + +Usage: + python rename_h5_groups.py file.h5 old_name new_name [--level shot|tree|signal] + python rename_h5_groups.py 200000.h5 d3d D3D --level tree + python rename_h5_groups.py 200000.h5 200000 200001 --level shot +""" + +import h5py +import sys +import argparse +import numpy as np +from pathlib import Path + + +def compare_groups(group1, group2, path=""): + """ + Recursively compare two HDF5 groups for equality. + + Args: + group1: First HDF5 group + group2: Second HDF5 group + path: Current path (for error messages) + + Returns: + (is_equal, differences) tuple + """ + differences = [] + + # Check if both have same keys + keys1 = set(group1.keys()) + keys2 = set(group2.keys()) + + if keys1 != keys2: + only_in_1 = keys1 - keys2 + only_in_2 = keys2 - keys1 + if only_in_1: + differences.append(f"{path}: Only in first: {only_in_1}") + if only_in_2: + differences.append(f"{path}: Only in second: {only_in_2}") + return False, differences + + # Compare each key + for key in keys1: + item1 = group1[key] + item2 = group2[key] + current_path = f"{path}/{key}" if path else key + + # Check if both are same type (group vs dataset) + if isinstance(item1, h5py.Group) != isinstance(item2, h5py.Group): + differences.append(f"{current_path}: Type mismatch") + return False, differences + + if isinstance(item1, h5py.Group): + # Recursively compare subgroups + equal, subdiffs = compare_groups(item1, item2, current_path) + if not equal: + differences.extend(subdiffs) + return False, differences + else: + # Compare datasets + try: + # Handle scalar vs array datasets + if item1.shape == (): + # Scalar dataset - use [()] instead of [:] + data1 = item1[()] + data2 = item2[()] + else: + # Array dataset + data1 = item1[:] + data2 = item2[:] + + # Check shapes + if isinstance(data1, np.ndarray) and isinstance(data2, np.ndarray): + if data1.shape != data2.shape: + differences.append( + f"{current_path}: Shape mismatch {data1.shape} vs {data2.shape}") + return False, differences + + # Check data equality (handle NaNs) + if isinstance(data1, (np.ndarray, np.floating, float)): + # For float data, use allclose and handle NaNs + if isinstance(data1, np.ndarray): + if np.issubdtype(data1.dtype, np.floating): + nan_mask1 = np.isnan(data1) + nan_mask2 = np.isnan(data2) + + if not np.array_equal(nan_mask1, nan_mask2): + differences.append( + f"{current_path}: NaN positions differ") + return False, differences + + # Compare non-NaN values + if np.any(nan_mask1): + # Has NaNs + if not np.allclose(data1[~nan_mask1], data2[~nan_mask2], + rtol=1e-9, atol=1e-9): + differences.append( + f"{current_path}: Data values differ") + return False, differences + else: + # No NaNs + if not np.allclose(data1, data2, rtol=1e-9, atol=1e-9): + differences.append( + f"{current_path}: Data values differ") + return False, differences + else: + # Non-float array + if not np.array_equal(data1, data2): + differences.append(f"{current_path}: Data values differ") + return False, differences + else: + # Scalar float/number + if np.isnan(data1) and np.isnan(data2): + pass # Both NaN, equal + elif np.isnan(data1) or np.isnan(data2): + differences.append( + f"{current_path}: One is NaN, other is not") + return False, differences + elif not np.isclose(data1, data2, rtol=1e-9, atol=1e-9): + differences.append( + f"{current_path}: Data values differ ({data1} vs {data2})") + return False, differences + elif isinstance(data1, bytes) and isinstance(data2, bytes): + # String/bytes comparison + if data1 != data2: + differences.append(f"{current_path}: String values differ") + return False, differences + else: + # General comparison + if data1 != data2: + differences.append(f"{current_path}: Data values differ") + return False, differences + + # Check attributes + attrs1 = dict(item1.attrs) + attrs2 = dict(item2.attrs) + + # Compare attributes (handle different types) + if set(attrs1.keys()) != set(attrs2.keys()): + differences.append(f"{current_path}: Attribute keys differ") + return False, differences + + for attr_key in attrs1.keys(): + val1 = attrs1[attr_key] + val2 = attrs2[attr_key] + + # Convert to comparable types + if isinstance(val1, bytes): + val1 = val1.decode('utf-8') if isinstance(val1, bytes) else val1 + if isinstance(val2, bytes): + val2 = val2.decode('utf-8') if isinstance(val2, bytes) else val2 + + if isinstance(val1, np.ndarray) and isinstance(val2, np.ndarray): + if not np.array_equal(val1, val2): + differences.append( + f"{current_path}: Attribute '{attr_key}' differs") + return False, differences + else: + if val1 != val2: + differences.append( + f"{current_path}: Attribute '{attr_key}' differs ({val1} vs {val2})") + return False, differences + + except Exception as e: + differences.append(f"{current_path}: Comparison error: {e}") + return False, differences + + # Check group-level attributes + attrs1 = dict(group1.attrs) + attrs2 = dict(group2.attrs) + + if set(attrs1.keys()) != set(attrs2.keys()): + differences.append(f"{path}: Group attribute keys differ") + return False, differences + + for attr_key in attrs1.keys(): + val1 = attrs1[attr_key] + val2 = attrs2[attr_key] + + if isinstance(val1, bytes): + val1 = val1.decode('utf-8') if isinstance(val1, bytes) else val1 + if isinstance(val2, bytes): + val2 = val2.decode('utf-8') if isinstance(val2, bytes) else val2 + + if isinstance(val1, np.ndarray) and isinstance(val2, np.ndarray): + if not np.array_equal(val1, val2): + differences.append(f"{path}: Group attribute '{attr_key}' differs") + return False, differences + else: + if val1 != val2: + differences.append(f"{path}: Group attribute '{attr_key}' differs") + return False, differences + + return True, [] + + +def rename_group(h5_file, old_name, new_name, level='tree', shot=None, tree=None, + dry_run=False, verbose=True): + """ + Rename a group in HDF5 file. + + If new_name already exists, compares data. If identical, removes old_name. + If different, raises error. + + Args: + h5_file: Path to HDF5 file + old_name: Current name of group + new_name: New name for group + level: 'shot', 'tree', or 'signal' + shot: Shot number (required for tree/signal level) + tree: Tree name (required for signal level) + dry_run: Show what would be renamed without doing it + verbose: Print progress + """ + if old_name == new_name: + if verbose: + print(f"Warning: old_name and new_name are identical ('{old_name}')") + print("Nothing to do.") + return 0 + + if not Path(h5_file).exists(): + print(f"Error: File does not exist: {h5_file}") + sys.exit(1) + + mode = 'r' if dry_run else 'a' + + with h5py.File(h5_file, mode) as f: + renamed = 0 + + if level == 'shot': + # Rename shot: 200000 -> 200001 + if old_name not in f: + print(f"Error: Shot '{old_name}' not found") + return 0 + + if new_name in f: + # Compare data + if verbose: + print(f"Shot '{new_name}' already exists, comparing data...") + + is_equal, differences = compare_groups(f[old_name], f[new_name], + old_name) + + if is_equal: + if verbose: + print(f" ✓ Data is identical") + print(f" Removing duplicate shot: {old_name}") + + if not dry_run: + del f[old_name] + + renamed = 1 + else: + print(f" ✗ Data is different:") + for diff in differences[:5]: # Show first 5 differences + print(f" - {diff}") + if len(differences) > 5: + print(f" ... and {len(differences) - 5} more differences") + print(f"Error: Cannot rename - data conflict") + return 0 + else: + # Normal rename + if verbose: + print(f"Renaming shot: {old_name} -> {new_name}") + + if not dry_run: + f.copy(f[old_name], f, new_name) + del f[old_name] + + renamed = 1 + + elif level == 'tree': + # Rename tree in all shots or specific shot + shots_to_process = [shot] if shot else list(f.keys()) + + for shot_name in shots_to_process: + if shot_name not in f: + print(f"Warning: Shot '{shot_name}' not found") + continue + + if old_name not in f[shot_name]: + if verbose: + print(f"Shot {shot_name}: Tree '{old_name}' not found") + continue + + old_path = f"{shot_name}/{old_name}" + new_path = f"{shot_name}/{new_name}" + + if new_name in f[shot_name]: + # Compare data + if verbose: + print( + f"Shot {shot_name}: Tree '{new_name}' already exists, comparing data...") + + is_equal, differences = compare_groups(f[old_path], f[new_path], + old_path) + + if is_equal: + if verbose: + print(f" ✓ Data is identical") + print(f" Removing duplicate tree: {old_name}") + + if not dry_run: + del f[old_path] + + renamed += 1 + else: + print(f" ✗ Data is different:") + for diff in differences[:5]: + print(f" - {diff}") + if len(differences) > 5: + print(f" ... and {len(differences) - 5} more differences") + print( + f"Error: Cannot rename tree in shot {shot_name} - data conflict") + else: + # Normal rename + if verbose: + print( + f"Shot {shot_name}: Renaming tree {old_name} -> {new_name}") + + if not dry_run: + f.copy(f[old_path], f[shot_name], new_name) + del f[old_path] + + renamed += 1 + + elif level == 'signal': + # Rename signal in specific tree + if not shot or not tree: + print("Error: --shot and --tree required for signal level") + return 0 + + tree_path = f"{shot}/{tree}" + + if tree_path not in f: + print(f"Error: Tree '{tree_path}' not found") + return 0 + + if old_name not in f[tree_path]: + print(f"Error: Signal '{old_name}' not found in {tree_path}") + return 0 + + old_path = f"{tree_path}/{old_name}" + new_path = f"{tree_path}/{new_name}" + + if new_name in f[tree_path]: + # Compare data + if verbose: + print(f"Signal '{new_name}' already exists, comparing data...") + + is_equal, differences = compare_groups(f[old_path], f[new_path], + old_path) + + if is_equal: + if verbose: + print(f" ✓ Data is identical") + print(f" Removing duplicate signal: {old_name}") + + if not dry_run: + del f[old_path] + + renamed = 1 + else: + print(f" ✗ Data is different:") + for diff in differences[:5]: + print(f" - {diff}") + if len(differences) > 5: + print(f" ... and {len(differences) - 5} more differences") + print(f"Error: Cannot rename signal - data conflict") + return 0 + else: + # Normal rename + if verbose: + print(f"Renaming signal: {old_path} -> {new_path}") + + if not dry_run: + f.copy(f[old_path], f[tree_path], new_name) + del f[old_path] + + renamed = 1 + + if verbose and renamed > 0: + if dry_run: + print(f"\nDry run: Would rename/remove {renamed} group(s)") + else: + print(f"\nRenamed/removed {renamed} group(s) in {h5_file}") + + return renamed + + +def batch_rename(h5_file, mapping_file, level='tree', dry_run=False, verbose=True): + """ + Rename multiple groups from a mapping file. + + Args: + h5_file: Path to HDF5 file + mapping_file: Path to text file with "old_name new_name" pairs + level: 'shot', 'tree', or 'signal' + dry_run: Show what would be renamed + verbose: Print progress + """ + if not Path(mapping_file).exists(): + print(f"Error: Mapping file does not exist: {mapping_file}") + sys.exit(1) + + total_renamed = 0 + + with open(mapping_file, 'r') as f: + for line in f: + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + parts = line.split() + if len(parts) != 2: + print(f"Warning: Invalid line (expected 'old new'): {line}") + continue + + old_name, new_name = parts + renamed = rename_group(h5_file, old_name, new_name, level=level, + dry_run=dry_run, verbose=verbose) + total_renamed += renamed + + print(f"\nTotal renamed/removed: {total_renamed}") + + +def main(): + parser = argparse.ArgumentParser( + description='Rename groups in HDF5 files (with data comparison)', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Rename a tree in all shots + python rename_h5_groups.py 200000.h5 d3d D3D --level tree + + # If D3D already exists and data is identical, d3d will be removed + # If D3D already exists and data differs, error is raised + + # Rename a tree in specific shot + python rename_h5_groups.py 200000.h5 d3d D3D --level tree --shot 200000 + + # Rename a shot + python rename_h5_groups.py data.h5 200000 200001 --level shot + + # Batch rename from file + python rename_h5_groups.py 200000.h5 --batch mapping.txt --level tree + + # Dry run (preview changes) + python rename_h5_groups.py 200000.h5 d3d D3D --level tree --dry-run + +Behavior when new name exists: + - Compares data recursively (structure, values, attributes) + - If identical: removes old name (consolidates duplicates) + - If different: raises error to prevent data loss + """ + ) + + parser.add_argument('file', help='HDF5 file to modify') + parser.add_argument('old_name', nargs='?', help='Current group name') + parser.add_argument('new_name', nargs='?', help='New group name') + parser.add_argument('--level', choices=['shot', 'tree', 'signal'], + default='tree', + help='What to rename (default: tree)') + parser.add_argument('--shot', help='Shot number (for tree/signal level)') + parser.add_argument('--tree', help='Tree name (for signal level)') + parser.add_argument('--batch', help='Batch rename from mapping file') + parser.add_argument('--dry-run', action='store_true', + help='Show what would be renamed without doing it') + parser.add_argument('-q', '--quiet', action='store_true', + help='Suppress progress messages') + + args = parser.parse_args() + + if args.batch: + # Batch mode + batch_rename(args.file, args.batch, level=args.level, + dry_run=args.dry_run, verbose=not args.quiet) + else: + # Single rename mode + if not args.old_name or not args.new_name: + parser.error("old_name and new_name required (or use --batch)") + + rename_group(args.file, args.old_name, args.new_name, + level=args.level, shot=args.shot, tree=args.tree, + dry_run=args.dry_run, verbose=not args.quiet) + + +if __name__ == '__main__': + main() diff --git a/scripts/data_preparation/convert_float64_to_float32.py b/scripts/data_preparation/convert_float64_to_float32.py new file mode 100644 index 0000000..869eedc --- /dev/null +++ b/scripts/data_preparation/convert_float64_to_float32.py @@ -0,0 +1,29 @@ +import h5py +import numpy as np +from pathlib import Path +from tqdm import tqdm + + +def convert_float64_to_float32(src_path, dst_path): + with h5py.File(src_path, "r") as src, h5py.File(dst_path, "w") as dst: + def copy_item(name, obj): + if isinstance(obj, h5py.Group): + dst.require_group(name) + elif isinstance(obj, h5py.Dataset): + data = obj[:] + if data.dtype == np.float64: + data = data.astype(np.float32) + dst.create_dataset(name, data=data) + + src.visititems(copy_item) + + +if __name__ == "__main__": + for k, filename in enumerate(tqdm(sorted(Path("/scratch/gpfs/EKOLEMEN/foundation_model/").glob("*_processed.h5")))): + if k <= 5500: + continue + src = filename + dst = filename.with_stem(src.stem + "_2") + convert_float64_to_float32(src, dst) + print(f"{src.stat().st_size / 1e9:.2f} GB → {dst.stat().st_size / 1e9:.2f} GB") + dst.rename(filename) From d9c8c156312bb7d975e5aa04a80f62a63f5632d7 Mon Sep 17 00:00:00 2001 From: renierts Date: Tue, 10 Mar 2026 15:32:08 -0400 Subject: [PATCH 30/30] Shell script for batch dtype conversion. --- scripts/slurm/convert_dtypes.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 scripts/slurm/convert_dtypes.sh diff --git a/scripts/slurm/convert_dtypes.sh b/scripts/slurm/convert_dtypes.sh new file mode 100755 index 0000000..d9930a4 --- /dev/null +++ b/scripts/slurm/convert_dtypes.sh @@ -0,0 +1,12 @@ +#!/bin/bash +#SBATCH --job-name=convert_dtype # create a short name for your job +#SBATCH --output=logs/convert_dtype.out +#SBATCH --error=logs/convert_dtype.err +#SBATCH --cpus-per-task=1 # cpu-cores per task (>1 if multi-threaded tasks) +#SBATCH --nodes=1 # node count +#SBATCH --mem-per-cpu=32G # memory per cpu-core (4G is default) +#SBATCH --time=12:00:00 # total run time limit (HH:MM:SS) +#SBATCH --mail-type=all # send email on job start, end and fault +#SBATCH --mail-user=ps9551@princeton.edu + +pixi run python -u ../data_preparation/convert_float64_to_float32.py