From d3887ad62904ba40fc7300ab724f59282f437e5f Mon Sep 17 00:00:00 2001 From: cwognum Date: Wed, 13 Mar 2024 10:46:22 -0400 Subject: [PATCH 1/9] Implemented major pieces for dataset factory and adapters A lot of clean up is still needed --- polaris/benchmark/_base.py | 29 +++-- polaris/dataset/__init__.py | 9 +- polaris/dataset/_adapters.py | 32 +++++ polaris/dataset/_dataset.py | 169 +++----------------------- polaris/dataset/_factories.py | 202 +++++++++++++++++++++++++++++++ polaris/dataset/_subset.py | 44 +++---- polaris/utils/types.py | 11 +- tests/test_dataset.py | 54 --------- tests/test_to_zarr_converters.py | 7 ++ 9 files changed, 300 insertions(+), 257 deletions(-) create mode 100644 polaris/dataset/_adapters.py create mode 100644 polaris/dataset/_factories.py create mode 100644 tests/test_to_zarr_converters.py diff --git a/polaris/benchmark/_base.py b/polaris/benchmark/_base.py index 895cf25e..beb499b9 100644 --- a/polaris/benchmark/_base.py +++ b/polaris/benchmark/_base.py @@ -27,7 +27,6 @@ from polaris.utils.misc import listit from polaris.utils.types import ( AccessType, - DataFormat, HubOwner, PredictionsType, SplitType, @@ -353,10 +352,7 @@ def task_type(self) -> TaskType: return v.value def get_train_test_split( - self, - input_format: DataFormat = "dict", - target_format: DataFormat = "dict", - featurization_fn: Optional[Callable] = None, + self, featurization_fn: Optional[Callable] = None ) -> tuple[Subset, Union["Subset", dict[str, Subset]]]: """Construct the train and test sets, given the split in the benchmark specification. @@ -365,8 +361,8 @@ def get_train_test_split( data-loaders on top of. Args: - input_format: How the input data is returned from the `Subset` object. - target_format: How the target data is returned from the `Subset` object. + input_adapter: How the input data is returned from the `Subset` object. + target_adapter: How the target data is returned from the `Subset` object. This will only affect the train set. featurization_fn: A function to apply to the input data. If a multi-input benchmark, this function expects an input in the format specified by the `input_format` parameter. @@ -382,9 +378,7 @@ def _get_subset(indices, hide_targets): dataset=self.dataset, indices=indices, input_cols=self.input_cols, - input_format=input_format, target_cols=self.target_cols, - target_format=target_format, hide_targets=hide_targets, featurization_fn=featurization_fn, ) @@ -459,7 +453,12 @@ def evaluate(self, y_pred: PredictionsType) -> BenchmarkResults: if not isinstance(y_true_subset, dict): # Single task score = metric(y_true=y_true_subset, y_pred=y_pred[test_label]) - scores.loc[len(scores)] = (test_label, self.target_cols[0], metric, score) + scores.loc[len(scores)] = ( + test_label, + self.target_cols[0], + metric, + score, + ) continue # Otherwise, for every target... @@ -467,7 +466,10 @@ def evaluate(self, y_pred: PredictionsType) -> BenchmarkResults: # Single-task metrics for a multi-task benchmark # In such a setting, there can be NaN values, which we thus have to filter out. mask = ~np.isnan(y_true_target) - score = metric(y_true=y_true_target[mask], y_pred=y_pred[test_label][target_label][mask]) + score = metric( + y_true=y_true_target[mask], + y_pred=y_pred[test_label][target_label][mask], + ) scores.loc[len(scores)] = (test_label, target_label, metric, score) return BenchmarkResults(results=scores, benchmark_name=self.name, benchmark_owner=self.owner) @@ -488,7 +490,10 @@ def upload_to_hub( from polaris.hub.client import PolarisHubClient with PolarisHubClient( - env_file=env_file, settings=settings, cache_auth_token=cache_auth_token, **kwargs + env_file=env_file, + settings=settings, + cache_auth_token=cache_auth_token, + **kwargs, ) as client: return client.upload_benchmark(self, access=access, owner=owner) diff --git a/polaris/dataset/__init__.py b/polaris/dataset/__init__.py index ecd3098b..4ef13fb8 100644 --- a/polaris/dataset/__init__.py +++ b/polaris/dataset/__init__.py @@ -1,9 +1,6 @@ -from polaris.dataset._column import ColumnAnnotation +from polaris.dataset._adapters import Adapter +from polaris.dataset._column import ColumnAnnotation, Modality from polaris.dataset._dataset import Dataset from polaris.dataset._subset import Subset -__all__ = [ - "ColumnAnnotation", - "Dataset", - "Subset", -] +__all__ = ["ColumnAnnotation", "Dataset", "Subset", "Modality", "Adapter"] diff --git a/polaris/dataset/_adapters.py b/polaris/dataset/_adapters.py new file mode 100644 index 00000000..e62fa029 --- /dev/null +++ b/polaris/dataset/_adapters.py @@ -0,0 +1,32 @@ +import abc +from typing import Optional + +import datamol as dm +from pydantic import BaseModel + + +class Adapter(BaseModel, abc.ABC): + column: Optional[str] = None + + def __call__(self, data: dict) -> dict: + v = data[self.column] + if isinstance(v, tuple): + data[self.column] = [self.adapt(x) for x in v] + else: + data[self.column] = self.adapt(v) + + return data + + @abc.abstractmethod + def adapt(self, data: dict): + raise NotImplementedError + + +class SmilesAdapter(Adapter): + def adapt(self, data: str) -> dm.Mol: + return dm.to_mol(data) + + +class MolBytestringAdapter(Adapter): + def adapt(self, data: bytes) -> dm.Mol: + return dm.Mol(data) diff --git a/polaris/dataset/_dataset.py b/polaris/dataset/_dataset.py index fb4c4dd8..18455fb1 100644 --- a/polaris/dataset/_dataset.py +++ b/polaris/dataset/_dataset.py @@ -2,7 +2,7 @@ import os.path from collections import defaultdict from hashlib import md5 -from typing import Dict, Literal, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union import fsspec import numpy as np @@ -17,6 +17,7 @@ ) from polaris._artifact import BaseArtifactModel +from polaris.dataset._adapters import Adapter from polaris.dataset._column import ColumnAnnotation from polaris.hub.settings import PolarisHubSettings from polaris.utils import fs @@ -30,7 +31,6 @@ _SUPPORTED_TABLE_EXTENSIONS = ["parquet"] _CACHE_SUBDIR = "datasets" _INDEX_SEP = "#" -_INDEX_FMT = f"{{path}}{_INDEX_SEP}{{index}}" class Dataset(BaseArtifactModel): @@ -68,6 +68,8 @@ class Dataset(BaseArtifactModel): # Public attributes # Data table: Union[pd.DataFrame, str] + input_adapter: Optional[Adapter] = None + target_adapter: Optional[Adapter] = None md5sum: Optional[str] = None # Additional meta-data @@ -193,11 +195,13 @@ def get_data(self, row: Union[str, int], col: str) -> np.ndarray: the content of the referenced file is loaded to memory. """ - def _load(p: str, index: Optional[int]) -> np.ndarray: + def _load(p: str, index: Optional[Union[int, slice]]) -> np.ndarray: """Tiny helper function to reduce code repetition.""" - arr = zarr.convenience.load(p) + arr = zarr.open(p, mode="r") if index is not None: arr = arr[index] + if isinstance(index, slice): + arr = tuple(arr) return arr value = self.table.loc[row, col] @@ -238,153 +242,13 @@ def upload_to_hub( from polaris.hub.client import PolarisHubClient with PolarisHubClient( - env_file=env_file, settings=settings, cache_auth_token=cache_auth_token, **kwargs + env_file=env_file, + settings=settings, + cache_auth_token=cache_auth_token, + **kwargs, ) as client: return client.upload_dataset(self, access=access, owner=owner) - @classmethod - def from_zarr(cls, path: str) -> "Dataset": - """Parse a [.zarr](https://zarr.readthedocs.io/en/stable/index.html) hierarchy into a Polaris `Dataset`. - - In short: A `.zarr` file can contain groups and arrays, where each group can again contain groups and arrays. - Additional user attributes (for any array or group) are saved as JSON files. - - Within Polaris: - - 1. Each subgroup of the root group corresponds to a single column. - 2. Each subgroup can contain: - - A single array with _all_ datapoints. - - A single array _per_ datapoint. - 3. Additional meta-data is saved to the user attributes of the root group. - 3. The indices are required to be integers. - - Tip: Tutorial - To learn more about the zarr format, see the - [tutorial](../tutorials/dataset_zarr.ipynb). - - Warning: Beta functionality - This feature is still in beta and the API will likely change. Please report any issues you encounter. - - Args: - path: The path to the root of the `.zarr` directory. Should be compatible with fsspec. - """ - - logger.warning( - "We are still testing to save and load from .zarr files. " - "This part of the API will likely change." - ) - - root = zarr.open(path, "r") - - # Get the user attributes - attrs = root.attrs.asdict() - - # TODO (cwognum): This is outdated and needs to be updated. - possible_user_attr = ["name", "description", "source", "annotations"] - attrs = {k: v for k, v in attrs.items() if k in possible_user_attr} - - # Set the annotations - attrs["annotations"] = attrs.get("annotations", {}) - for column_label in root.group_keys(): - obj = attrs["annotations"].get(column_label, {}) - obj = ColumnAnnotation.model_validate(obj) - obj.is_pointer = True - attrs["annotations"][column_label] = obj - - # Construct the table - # Parse any group into a column - data = defaultdict(dict) - for col, group in root.groups(): - keys = list(group.array_keys()) - - if len(keys) == 1: - arr = group[keys[0]] - for i, arr_row in enumerate(arr): - # In case all data is saved in a single array, we construct a path with an index suffix. - data[col][i] = _INDEX_FMT.format(path=fs.join(path, arr.name), index=i) - - else: - for name, arr in group.arrays(): - try: - name = int(name) - except ValueError as error: - raise InvalidDatasetError( - "All names for arrays in the .zarr archive are required to be integers." - ) from error - data[col][name] = fs.join(path, arr.path) - - # Construct the dataset - table = pd.DataFrame(data) - return cls(table=table, **attrs) - - def to_zarr( - self, - destination: str, - array_mode: Dict[str, Literal["single", "multiple"]], - ) -> str: - """Saves a dataset to a .zarr file. For more information on the resulting structure, - see [`from_zarr`][polaris.dataset.Dataset.from_zarr]. - - Tip: Tutorial - To learn more about the zarr format, see the - [tutorial](../tutorials/dataset_zarr.ipynb). - - Warning: Beta functionality - This feature is still in beta and the API will likely change. Please report any issues you encounter. - - Args: - destination: The _directory_ to save the associated data to. - array_mode: For each of the columns, whether to save all datapoints in a single array - or create an array per datapoint. Should be one of "single" or "multiple". - - Returns: - The path to the root zarr file. - """ - - logger.warning( - "We are still testing to save and load from .zarr files. " - "This part of the API will likely change." - ) - - if array_mode not in ["single", "multiple"]: - raise ValueError(f"array_mode should be one of 'single' or 'multiple', not {array_mode}") - - fs.mkdir(destination, exist_ok=True) - path = fs.join(destination, "dataset.zarr") - - if not isinstance(array_mode, dict): - array_mode = {k: array_mode for k in self.table.columns} - - root = zarr.open(path, "w") - for col in self.table.columns: - group = root.create_group(col) - - # Load an example to get the dtype and shape - example = self.get_data(row=0, col=col) - - if array_mode[col] == "single": - # Create one big array for all datapoints - shape = (len(self.table), *example.shape) - arr = group.empty(col, shape=shape, dtype=example.dtype) - - for row in self.table.index: - # Save the data to the array - arr[row] = self.get_data(row=row, col=col) - else: - for row in self.table.index: - # Create an array per datapoint - group.array(row, self.get_data(row=row, col=col)) - - # Save the meta-data - # TODO (cwognum): This is outdated and needs to be updated. - root.user_attrs = { - "name": self.name, - "description": self.description, - "source": self.source, - "annotations": {k: v.model_dump() for k, v in self.annotations.items()}, - } - return path - @classmethod def from_json(cls, path: str): """Loads a benchmark from a JSON file. @@ -490,7 +354,14 @@ def _split_index_from_path(self, path: str) -> Tuple[str, Optional[int]]: index = None if _INDEX_SEP in path: path, index = path.split(_INDEX_SEP) - index = int(index) + index = index.split(":") + + if len(index) == 1: + index = int(index[0]) + elif len(index) == 2: + index = slice(int(index[0]), int(index[1])) + else: + raise ValueError(f"Invalid index format: {index}") return path, index def _copy_and_update_pointers( diff --git a/polaris/dataset/_factories.py b/polaris/dataset/_factories.py new file mode 100644 index 00000000..81fc8448 --- /dev/null +++ b/polaris/dataset/_factories.py @@ -0,0 +1,202 @@ +import abc +import os +import uuid +from typing import Dict, Optional, Tuple, TypeAlias + +import datamol as dm +import pandas as pd +import zarr +from rdkit import Chem + +from polaris.dataset import ColumnAnnotation, Dataset, Modality + +FactoryProduct: TypeAlias = Tuple[pd.DataFrame, Dict[str, ColumnAnnotation]] + + +class DatasetFactory: + """ + The DatasetFactory is meant to more easily create complex datasets. + It uses the factory design pattern. + """ + + def __init__(self, zarr_root_path: Optional[str] = None) -> None: + self.zarr_root_path = os.path.abspath(zarr_root_path) + self._zarr_root = None + + self.table: pd.DataFrame = pd.DataFrame() + self.annotations: Dict[str, ColumnAnnotation] = {} + + self._converters = {} + + @property + def zarr_root(self) -> zarr.Group: + if self.zarr_root_path is None: + raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") + + if self._zarr_root is None: + self._zarr_root = zarr.open(self.zarr_root_path, "w") + if not isinstance(self._zarr_root, zarr.Group): + raise ValueError("The root of the zarr hierarchy should be a group") + return self._zarr_root + + def register_converter(self, ext: str, converter): + self._converters[ext] = converter + + def reset(self): + self.table = pd.DataFrame() + self.annotations = {} + + def add_column( + self, + column: pd.Series, + annotation: Optional[ColumnAnnotation] = None, + ): + """Adds a single column""" + if column.name is None: + raise RuntimeError("You need to specify a column name") + + if annotation is not None and annotation.is_pointer: + if self.zarr_root is None: + raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") + + self.table[column.name] = column + + if annotation is None: + annotation = ColumnAnnotation() + self.annotations[column.name] = annotation + + def add_from_file(self, path: str): + """ """ + ext = dm.fs.get_extension(path) + converter = self._converters.get(ext) + if converter is None: + raise ValueError(f"No converter found for extension {ext}") + + table, annotations = converter.convert(path, self) + + for name, series in table.items(): + self.add_column(series, annotations.get(name)) + + def build(self) -> Dataset: + return Dataset(table=self.table, annotations=self.annotations) + + +class Converter(abc.ABC): + @abc.abstractmethod + def convert(self, path: str) -> FactoryProduct: + raise NotImplementedError + + +class SDFConverter(Converter): + """Convert from a SDF file""" + + def __init__( + self, + mol_column: str = "molecule", + smiles_column: Optional[str] = "smiles", + mol_id_column: Optional[str] = None, + mol_prop_as_cols: bool = True, + groupby_key: Optional[str] = None, + n_jobs: int = 1, + ) -> None: + """ """ + super().__init__() + self.mol_column = mol_column + self.smiles_column = smiles_column + self.mol_id_column = mol_id_column + self.mol_prop_as_cols = mol_prop_as_cols + self.groupby_key = groupby_key + self.n_jobs = n_jobs + + def convert(self, path: str, factory: DatasetFactory) -> FactoryProduct: + """ + Converts the molecules in an SDF file to a Polaris compatible format. + """ + + tmp_col = uuid.uuid4().hex + + # We do not sanitize the molecules or remove the Hs. + # We assume the SDF has been processed by the user already and do not want to change it. + df = dm.read_sdf( + path, + as_df=self.mol_prop_as_cols, + smiles_column=self.smiles_column, + mol_column=tmp_col, + remove_hs=False, + sanitize=False, + max_num_mols=1000, + ) + + if not isinstance(df, pd.DataFrame): + df = pd.DataFrame({tmp_col: df}) + + if self.mol_column in df.columns: + raise ValueError( + f"The column name '{self.mol_column}' clashes with the name of a property in the SDF file. " + f"Please choose another name by setting the `mol_column` in the {self.__class__.__name__}." + ) + + # Add a column with the molecule name if it doesn't exist yet + if self.mol_id_column is not None and self.mol_id_column not in df.columns: + + def _get_name(mol: dm.Mol): + return mol.GetProp(self.mol_id_column) if mol.HasProp(self.mol_id_column) else None + + names = dm.parallelized(_get_name, df[tmp_col], n_jobs=self.n_jobs, scheduler="threads") + df[self.mol_id_column] = names + + # Add a column with the SMILES if it doesn't exist yet + if self.smiles_column is not None and self.smiles_column not in df.columns: + names = dm.parallelized(dm.to_smiles, df[tmp_col], n_jobs=self.n_jobs) + df[self.smiles_column] = names + + # Convert the molecules to binary strings (for ML purposes, this should be lossless). + # This might not be the most storage efficient, but is fastest and easiest to maintain. + # We do not save the MolProps, because we have already extracted these into columns. + # See: https://github.com/rdkit/rdkit/discussions/7235 + props = Chem.PropertyPickleOptions.AllProps + if self.mol_prop_as_cols: + props &= ~Chem.PropertyPickleOptions.MolProps + bytes_data = [mol.ToBinary(props) for mol in df[tmp_col]] + + df.drop(columns=[tmp_col], inplace=True) + + # Create the zarr array + factory.zarr_root.array(self.mol_column, bytes_data, dtype=bytes) + + # Add a pointer column to the table + # We support grouping by a key, to allow inputs of variable length + + grouped = pd.DataFrame(columns=[*df.columns, self.mol_column]) + if self.groupby_key is not None: + for _, group in df.reset_index(drop=True).groupby(by=self.groupby_key): + start = group.index[0] + end = group.index[-1] + + if group.nunique().sum() != len(group.columns): + raise ValueError( + f"After grouping by {self.groupby_key}, values for other columns are not unique within a group. " + "Please handle this manually to ensure aggregation is done correctly." + ) + + # Get the pointer path + pointer_idx = f"{start}:{end}" if start != end else f"{start}" + pointer = f"{factory.zarr_root_path}/{self.mol_column}#{pointer_idx}" + + # Get the single unique value per column for the group and append + unique_values = [group[col].unique()[0] for col in df.columns] + grouped.loc[len(grouped)] = [*unique_values, pointer] + + df = grouped + + else: + pointers = [f"{factory.zarr_root_path}/{self.mol_column}#{i}" for i in range(len(df))] + df[self.mol_column] = pd.Series(pointers) + + # Set the annotations + annotations = {self.mol_column: ColumnAnnotation(is_pointer=True, modality=Modality.MOLECULE_3D)} + if self.smiles_column is not None: + annotations[self.smiles_column] = ColumnAnnotation(modality=Modality.MOLECULE) + + # Return the dataframe and the annotations + return df, annotations diff --git a/polaris/dataset/_subset.py b/polaris/dataset/_subset.py index 04f5df95..756465a8 100644 --- a/polaris/dataset/_subset.py +++ b/polaris/dataset/_subset.py @@ -2,9 +2,9 @@ import numpy as np -from polaris.dataset import Dataset +from polaris.dataset import Adapter, Dataset from polaris.utils.errors import TestAccessError -from polaris.utils.types import DataFormat, DatapointType +from polaris.utils.types import DatapointType class Subset: @@ -64,8 +64,8 @@ def __init__( indices: List[Union[int, Sequence[int]]], input_cols: Union[List[str], str], target_cols: Union[List[str], str], - input_format: DataFormat = "dict", - target_format: DataFormat = "dict", + input_adapter: Optional[Adapter] = None, + target_adapter: Optional[Adapter] = None, featurization_fn: Optional[Callable] = None, hide_targets: bool = False, ): @@ -73,8 +73,8 @@ def __init__( self.indices = indices self.target_cols = target_cols if isinstance(target_cols, list) else [target_cols] self.input_cols = input_cols if isinstance(input_cols, list) else [input_cols] - self._input_format = input_format - self._target_format = target_format + self._input_adapter = input_adapter + self._target_adapter = target_adapter self._featurization_fn = featurization_fn @@ -112,24 +112,12 @@ def y(self): """Alias for `self.as_array("y")`""" return self.as_array("y") - @staticmethod - def _format(data: dict, order: List[str], fmt: str): - """ - Converts the internally used dict format to the user-specified format. - If the user-specified format is a tuple, it orders the column according to the specified order. - """ - if len(data) == 1: - data = list(data.values())[0] - elif fmt == "tuple": - data = tuple(data[k] for k in order) - return data - def _get_single( self, row: str | int, cols: List[str], featurization_fn: Optional[Callable], - format: DataFormat, + adapter: Optional[Adapter], ): """ Loads a subset of the variables for a single data-point from the datasets. @@ -139,14 +127,17 @@ def _get_single( row: The row index of the datapoint. cols: The columns (i.e. variables) to load for that data point. featurization_fn: The transformation function to apply to the data-point. - format: The format to return the data-point in. + adapter: Format the data-point to a specific format. """ # Load the data-point # Also handles loading data stored in external files for pointer columns ret = {col: self.dataset.get_data(row, col) for col in cols} # Format - ret = self._format(ret, cols, format) + if adapter is not None: + ret = adapter(ret) + if len(ret) == 1: + ret = ret[cols[0]] # Featurize if featurization_fn is not None: @@ -156,11 +147,11 @@ def _get_single( def _get_single_input(self, row: str | int): """Get a single input for a specific data-point and given the benchmark specification.""" - return self._get_single(row, self.input_cols, self._featurization_fn, self._input_format) + return self._get_single(row, self.input_cols, self._featurization_fn, self._input_adapter) def _get_single_output(self, row: str | int): """Get a single output for a specific data-point and given the benchmark specification.""" - return self._get_single(row, self.target_cols, None, self._target_format) + return self._get_single(row, self.target_cols, None, self._target_adapter) def as_array(self, data_type: Union[Literal["x"], Literal["y"], Literal["xy"]]): """ @@ -187,13 +178,10 @@ def as_array(self, data_type: Union[Literal["x"], Literal["y"], Literal["xy"]]): # If the return format is a dict, we want to convert # from an array of dicts to a dict of arrays. - if data_type == "y" and self._target_format == "dict": + if data_type == "y": ret = {k: np.array([v[k] for v in ret]) for k in self.target_cols} - elif data_type == "x" and self._input_format == "dict": + elif data_type == "x": ret = {k: np.array([v[k] for v in ret]) for k in self.input_cols} - else: - # The format is a tuple, so we have list of tuples and convert this to an array - ret = np.array(ret) return ret diff --git a/polaris/utils/types.py b/polaris/utils/types.py index 19591a48..0cb3ba92 100644 --- a/polaris/utils/types.py +++ b/polaris/utils/types.py @@ -45,11 +45,6 @@ - No target, a single target or a multiple targets (either as dict or tuple) """ -DataFormat: TypeAlias = Literal["dict", "tuple"] -""" -The target formats that are supported by the `Subset` class. -""" - SlugStringType: TypeAlias = Annotated[ str, StringConstraints(pattern="^[a-z0-9-]+$", min_length=4, max_length=64) ] @@ -129,9 +124,9 @@ class License(BaseModel): Else it is required to manually specify this. """ - SPDX_LICENSE_DATA_PATH: ClassVar[str] = ( - "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" - ) + SPDX_LICENSE_DATA_PATH: ClassVar[ + str + ] = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" id: str reference: Optional[HttpUrlString] = None diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 01922720..eb1b3965 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -99,29 +99,6 @@ def _check_for_failure(_kwargs): assert dataset.md5sum is not None -@pytest.mark.parametrize("array_per_datapoint", [True, False]) -def test_dataset_from_zarr( - test_zarr_archive_single_array, test_zarr_archive_multiple_arrays, array_per_datapoint -): - """Test whether loading works when the zarr archive contains a single array or multiple arrays.""" - archive = test_zarr_archive_multiple_arrays if array_per_datapoint else test_zarr_archive_single_array - dataset = Dataset.from_zarr(archive) - assert len(dataset.table) == 100 - for i in range(100): - assert dataset.get_data(row=i, col="A").shape == (2048,) - assert dataset.get_data(row=i, col="B").shape == (2048,) - - -def test_dataset_from_zarr_equality(test_zarr_archive_single_array, test_zarr_archive_multiple_arrays): - """ - Test whether two methods for specifying .zarr datasets lead to the same dataset. - This specifically tests whether indexing a single arrow with our custom path syntax works. - """ - dataset_1 = Dataset.from_zarr(test_zarr_archive_single_array) - dataset_2 = Dataset.from_zarr(test_zarr_archive_multiple_arrays) - assert _equality_test(dataset_1, dataset_2) - - def test_dataset_from_json(test_dataset, tmpdir): """Test whether the dataset can be saved and loaded from json.""" test_dataset.to_json(str(tmpdir)) @@ -135,37 +112,6 @@ def test_dataset_from_json(test_dataset, tmpdir): assert _equality_test(test_dataset, new_dataset) -@pytest.mark.parametrize("array_per_datapoint", [True, False]) -def test_dataset_from_zarr_to_json_and_back( - test_zarr_archive_single_array, - test_zarr_archive_multiple_arrays, - array_per_datapoint, - tmpdir, -): - """ - Test whether a dataset with pointer columns, instantiated from a zarr archive, - can be saved to and loaded from json. - """ - - tmpdir = str(tmpdir) - json_dir = fs.join(tmpdir, "json") - zarr_dir = fs.join(tmpdir, "zarr") - - archive = test_zarr_archive_multiple_arrays if array_per_datapoint else test_zarr_archive_single_array - dataset = Dataset.from_zarr(archive) - path = dataset.to_json(json_dir) - - new_dataset = Dataset.from_json(path) - assert _equality_test(dataset, new_dataset) - - new_dataset = load_dataset(path) - assert _equality_test(dataset, new_dataset) - - path = new_dataset.to_zarr(zarr_dir, "multiple" if array_per_datapoint else "single") - new_dataset = load_dataset(path) - assert _equality_test(dataset, new_dataset) - - @pytest.mark.parametrize("array_per_datapoint", [True, False]) def test_dataset_caching( test_zarr_archive_single_array, diff --git a/tests/test_to_zarr_converters.py b/tests/test_to_zarr_converters.py new file mode 100644 index 00000000..4953e9e9 --- /dev/null +++ b/tests/test_to_zarr_converters.py @@ -0,0 +1,7 @@ +import datamol as dm + + +def test_sdf_bytestring_compat(tmpdir): + dm.Mol + print(tmpdir) + pass From ed714a1f82474a567960e6ad739a1d9b88db8add Mon Sep 17 00:00:00 2001 From: cwognum Date: Wed, 13 Mar 2024 14:35:58 -0400 Subject: [PATCH 2/9] Added a converter for Zarr files and fixed test cases --- polaris/benchmark/_base.py | 5 +- polaris/dataset/__init__.py | 11 ++- polaris/dataset/_adapters.py | 6 +- polaris/dataset/_dataset.py | 24 +++-- polaris/dataset/_factory.py | 91 ++++++++++++++++++ polaris/dataset/_subset.py | 18 ++-- polaris/dataset/converters/__init__.py | 5 + polaris/dataset/converters/_base.py | 32 +++++++ .../{_factories.py => converters/_sdf.py} | 94 +++---------------- polaris/dataset/converters/_zarr.py | 50 ++++++++++ tests/conftest.py | 23 +---- tests/test_dataset.py | 71 +++++++++----- tests/test_subset.py | 39 -------- tests/test_to_zarr_converters.py | 1 + 14 files changed, 279 insertions(+), 191 deletions(-) create mode 100644 polaris/dataset/_factory.py create mode 100644 polaris/dataset/converters/__init__.py create mode 100644 polaris/dataset/converters/_base.py rename polaris/dataset/{_factories.py => converters/_sdf.py} (61%) create mode 100644 polaris/dataset/converters/_zarr.py diff --git a/polaris/benchmark/_base.py b/polaris/benchmark/_base.py index beb499b9..b3170a52 100644 --- a/polaris/benchmark/_base.py +++ b/polaris/benchmark/_base.py @@ -361,9 +361,6 @@ def get_train_test_split( data-loaders on top of. Args: - input_adapter: How the input data is returned from the `Subset` object. - target_adapter: How the target data is returned from the `Subset` object. - This will only affect the train set. featurization_fn: A function to apply to the input data. If a multi-input benchmark, this function expects an input in the format specified by the `input_format` parameter. @@ -419,7 +416,7 @@ def evaluate(self, y_pred: PredictionsType) -> BenchmarkResults: # Instead of having the user pass the ground truth, we extract it from the benchmark spec ourselves. # This simplifies the API, but also was added to make accidental access to the test set targets less likely. # See also the `hide_targets` parameter in the `Subset` class. - test = self.get_train_test_split(target_format="dict")[1] + test = self.get_train_test_split()[1] if not isinstance(test, dict): test = {"test": test} diff --git a/polaris/dataset/__init__.py b/polaris/dataset/__init__.py index 4ef13fb8..198b28f6 100644 --- a/polaris/dataset/__init__.py +++ b/polaris/dataset/__init__.py @@ -1,6 +1,15 @@ from polaris.dataset._adapters import Adapter from polaris.dataset._column import ColumnAnnotation, Modality from polaris.dataset._dataset import Dataset +from polaris.dataset._factory import DatasetFactory, get_dataset_from_file from polaris.dataset._subset import Subset -__all__ = ["ColumnAnnotation", "Dataset", "Subset", "Modality", "Adapter"] +__all__ = [ + "ColumnAnnotation", + "Dataset", + "Subset", + "Modality", + "Adapter", + "DatasetFactory", + "get_dataset_from_file", +] diff --git a/polaris/dataset/_adapters.py b/polaris/dataset/_adapters.py index e62fa029..6e9311ff 100644 --- a/polaris/dataset/_adapters.py +++ b/polaris/dataset/_adapters.py @@ -1,20 +1,20 @@ import abc -from typing import Optional import datamol as dm from pydantic import BaseModel class Adapter(BaseModel, abc.ABC): - column: Optional[str] = None + column: str def __call__(self, data: dict) -> dict: + if self.column not in data: + return data v = data[self.column] if isinstance(v, tuple): data[self.column] = [self.adapt(x) for x in v] else: data[self.column] = self.adapt(v) - return data @abc.abstractmethod diff --git a/polaris/dataset/_dataset.py b/polaris/dataset/_dataset.py index 18455fb1..a0ccd522 100644 --- a/polaris/dataset/_dataset.py +++ b/polaris/dataset/_dataset.py @@ -2,7 +2,7 @@ import os.path from collections import defaultdict from hashlib import md5 -from typing import Dict, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Union import fsspec import numpy as np @@ -48,6 +48,8 @@ class Dataset(BaseArtifactModel): Attributes: table: The core data-structure, storing data-points in a row-wise manner. Can be specified as either a path to a `.parquet` file or a `pandas.DataFrame`. + default_adapters: The adapters that the Dataset recommends to use by default to change the format of the data + for specific columns. md5sum: The checksum is used to verify the version of the dataset specification. If specified, it will raise an error if the specified checksum doesn't match the computed checksum. readme: Markdown text that can be used to provide a formatted description of the dataset. @@ -68,8 +70,7 @@ class Dataset(BaseArtifactModel): # Public attributes # Data table: Union[pd.DataFrame, str] - input_adapter: Optional[Adapter] = None - target_adapter: Optional[Adapter] = None + default_adapters: Optional[List[Adapter]] = None md5sum: Optional[str] = None # Additional meta-data @@ -106,7 +107,13 @@ def _validate_model(cls, m: "Dataset"): # Verify that all annotations are for columns that exist if any(k not in m.table.columns for k in m.annotations): - raise InvalidDatasetError("There is annotations for columns that do not exist") + raise InvalidDatasetError("There are annotations for columns that do not exist") + + # Verify that all adapters are for columns that exist + if m.default_adapters is not None and any( + adapter.column not in m.table.columns for adapter in m.default_adapters + ): + raise InvalidDatasetError("There are default adapters for columns that do not exist") # Set a default for missing annotations and convert strings to Modality for c in m.table.columns: @@ -195,13 +202,12 @@ def get_data(self, row: Union[str, int], col: str) -> np.ndarray: the content of the referenced file is loaded to memory. """ - def _load(p: str, index: Optional[Union[int, slice]]) -> np.ndarray: + def _load(p: str, index: Union[int, slice]) -> np.ndarray: """Tiny helper function to reduce code repetition.""" arr = zarr.open(p, mode="r") - if index is not None: - arr = arr[index] - if isinstance(index, slice): - arr = tuple(arr) + arr = arr[index] + if isinstance(index, slice): + arr = tuple(arr) return arr value = self.table.loc[row, col] diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py new file mode 100644 index 00000000..5a6a5124 --- /dev/null +++ b/polaris/dataset/_factory.py @@ -0,0 +1,91 @@ +import os +from typing import Dict, Optional + +import datamol as dm +import pandas as pd +import zarr + +from polaris.dataset import ColumnAnnotation, Dataset +from polaris.dataset.converters import SDFConverter, ZarrConverter + + +def get_dataset_from_file(path: str, zarr_root_path: Optional[str] = None) -> Dataset: + """ + This function is a convenience function to create a dataset from a file. + It uses the factory design pattern to create the dataset. + For more complicated datasets, please use the `DatasetFactory` directly. + """ + factory = DatasetFactory(zarr_root_path=zarr_root_path) + factory.register_converter("sdf", SDFConverter()) + factory.register_converter("zarr", ZarrConverter()) + + factory.add_from_file(path) + return factory.build() + + +class DatasetFactory: + """ + The DatasetFactory is meant to more easily create complex datasets. + It uses the factory design pattern. + """ + + def __init__(self, zarr_root_path: Optional[str] = None) -> None: + self.zarr_root_path = os.path.abspath(zarr_root_path).rstrip("/") + self._zarr_root = None + + self.table: pd.DataFrame = pd.DataFrame() + self.annotations: Dict[str, ColumnAnnotation] = {} + + self._converters = {} + + @property + def zarr_root(self) -> zarr.Group: + if self.zarr_root_path is None: + raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") + + if self._zarr_root is None: + self._zarr_root = zarr.open(self.zarr_root_path, "w") + if not isinstance(self._zarr_root, zarr.Group): + raise ValueError("The root of the zarr hierarchy should be a group") + return self._zarr_root + + def register_converter(self, ext: str, converter): + self._converters[ext] = converter + + def reset(self): + self.table = pd.DataFrame() + self.annotations = {} + + def add_column( + self, + column: pd.Series, + annotation: Optional[ColumnAnnotation] = None, + ): + """Adds a single column""" + if column.name is None: + raise RuntimeError("You need to specify a column name") + + if annotation is not None and annotation.is_pointer: + if self.zarr_root is None: + raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") + + self.table[column.name] = column + + if annotation is None: + annotation = ColumnAnnotation() + self.annotations[column.name] = annotation + + def add_from_file(self, path: str): + """ """ + ext = dm.fs.get_extension(path) + converter = self._converters.get(ext) + if converter is None: + raise ValueError(f"No converter found for extension {ext}") + + table, annotations = converter.convert(path, self) + + for name, series in table.items(): + self.add_column(series, annotations.get(name)) + + def build(self) -> Dataset: + return Dataset(table=self.table, annotations=self.annotations) diff --git a/polaris/dataset/_subset.py b/polaris/dataset/_subset.py index 756465a8..ee70089d 100644 --- a/polaris/dataset/_subset.py +++ b/polaris/dataset/_subset.py @@ -64,8 +64,7 @@ def __init__( indices: List[Union[int, Sequence[int]]], input_cols: Union[List[str], str], target_cols: Union[List[str], str], - input_adapter: Optional[Adapter] = None, - target_adapter: Optional[Adapter] = None, + adapters: Optional[List[Adapter]] = None, featurization_fn: Optional[Callable] = None, hide_targets: bool = False, ): @@ -73,9 +72,8 @@ def __init__( self.indices = indices self.target_cols = target_cols if isinstance(target_cols, list) else [target_cols] self.input_cols = input_cols if isinstance(input_cols, list) else [input_cols] - self._input_adapter = input_adapter - self._target_adapter = target_adapter + self._adapters = self.dataset.default_adapters if adapters is None else adapters self._featurization_fn = featurization_fn # For the iterator implementation @@ -117,7 +115,6 @@ def _get_single( row: str | int, cols: List[str], featurization_fn: Optional[Callable], - adapter: Optional[Adapter], ): """ Loads a subset of the variables for a single data-point from the datasets. @@ -127,15 +124,16 @@ def _get_single( row: The row index of the datapoint. cols: The columns (i.e. variables) to load for that data point. featurization_fn: The transformation function to apply to the data-point. - adapter: Format the data-point to a specific format. """ # Load the data-point # Also handles loading data stored in external files for pointer columns ret = {col: self.dataset.get_data(row, col) for col in cols} # Format - if adapter is not None: - ret = adapter(ret) + if self._adapters is not None: + for adapter in self._adapters: + ret = adapter(ret) + if len(ret) == 1: ret = ret[cols[0]] @@ -147,11 +145,11 @@ def _get_single( def _get_single_input(self, row: str | int): """Get a single input for a specific data-point and given the benchmark specification.""" - return self._get_single(row, self.input_cols, self._featurization_fn, self._input_adapter) + return self._get_single(row, self.input_cols, self._featurization_fn) def _get_single_output(self, row: str | int): """Get a single output for a specific data-point and given the benchmark specification.""" - return self._get_single(row, self.target_cols, None, self._target_adapter) + return self._get_single(row, self.target_cols, None) def as_array(self, data_type: Union[Literal["x"], Literal["y"], Literal["xy"]]): """ diff --git a/polaris/dataset/converters/__init__.py b/polaris/dataset/converters/__init__.py new file mode 100644 index 00000000..b21882be --- /dev/null +++ b/polaris/dataset/converters/__init__.py @@ -0,0 +1,5 @@ +from polaris.dataset.converters._base import Converter +from polaris.dataset.converters._sdf import SDFConverter +from polaris.dataset.converters._zarr import ZarrConverter + +__all__ = ["Converter", "SDFConverter", "ZarrConverter"] diff --git a/polaris/dataset/converters/_base.py b/polaris/dataset/converters/_base.py new file mode 100644 index 00000000..cc87d8b2 --- /dev/null +++ b/polaris/dataset/converters/_base.py @@ -0,0 +1,32 @@ +import abc +from typing import Dict, Tuple, TypeAlias, Union + +import pandas as pd + +from polaris.dataset import ColumnAnnotation +from polaris.dataset._dataset import _INDEX_SEP + +FactoryProduct: TypeAlias = Tuple[pd.DataFrame, Dict[str, ColumnAnnotation]] + + +class Converter(abc.ABC): + @abc.abstractmethod + def convert(self, path: str) -> FactoryProduct: + """This converts a file into a table and possibly annotations""" + raise NotImplementedError + + @staticmethod + def get_pointer(root: str, column: str, index: Union[int, slice]) -> str: + """ + Creates a pointer. + + Args: + root: The root path of the zarr hierarchy. + column: The name of the column. Each column has its own group in the root. + index: The index or slice of the pointer. + """ + if isinstance(index, slice): + index_substr = f"{_INDEX_SEP}{index.start}:{index.stop}" + else: + index_substr = f"{_INDEX_SEP}{index}" + return f"{root}{column}{index_substr}" diff --git a/polaris/dataset/_factories.py b/polaris/dataset/converters/_sdf.py similarity index 61% rename from polaris/dataset/_factories.py rename to polaris/dataset/converters/_sdf.py index 81fc8448..ca5eb80f 100644 --- a/polaris/dataset/_factories.py +++ b/polaris/dataset/converters/_sdf.py @@ -1,90 +1,15 @@ -import abc -import os import uuid -from typing import Dict, Optional, Tuple, TypeAlias +from typing import TYPE_CHECKING, Optional import datamol as dm import pandas as pd -import zarr from rdkit import Chem -from polaris.dataset import ColumnAnnotation, Dataset, Modality +from polaris.dataset import ColumnAnnotation, Modality +from polaris.dataset.converters._base import Converter, FactoryProduct -FactoryProduct: TypeAlias = Tuple[pd.DataFrame, Dict[str, ColumnAnnotation]] - - -class DatasetFactory: - """ - The DatasetFactory is meant to more easily create complex datasets. - It uses the factory design pattern. - """ - - def __init__(self, zarr_root_path: Optional[str] = None) -> None: - self.zarr_root_path = os.path.abspath(zarr_root_path) - self._zarr_root = None - - self.table: pd.DataFrame = pd.DataFrame() - self.annotations: Dict[str, ColumnAnnotation] = {} - - self._converters = {} - - @property - def zarr_root(self) -> zarr.Group: - if self.zarr_root_path is None: - raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") - - if self._zarr_root is None: - self._zarr_root = zarr.open(self.zarr_root_path, "w") - if not isinstance(self._zarr_root, zarr.Group): - raise ValueError("The root of the zarr hierarchy should be a group") - return self._zarr_root - - def register_converter(self, ext: str, converter): - self._converters[ext] = converter - - def reset(self): - self.table = pd.DataFrame() - self.annotations = {} - - def add_column( - self, - column: pd.Series, - annotation: Optional[ColumnAnnotation] = None, - ): - """Adds a single column""" - if column.name is None: - raise RuntimeError("You need to specify a column name") - - if annotation is not None and annotation.is_pointer: - if self.zarr_root is None: - raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") - - self.table[column.name] = column - - if annotation is None: - annotation = ColumnAnnotation() - self.annotations[column.name] = annotation - - def add_from_file(self, path: str): - """ """ - ext = dm.fs.get_extension(path) - converter = self._converters.get(ext) - if converter is None: - raise ValueError(f"No converter found for extension {ext}") - - table, annotations = converter.convert(path, self) - - for name, series in table.items(): - self.add_column(series, annotations.get(name)) - - def build(self) -> Dataset: - return Dataset(table=self.table, annotations=self.annotations) - - -class Converter(abc.ABC): - @abc.abstractmethod - def convert(self, path: str) -> FactoryProduct: - raise NotImplementedError +if TYPE_CHECKING: + from polaris.dataset import DatasetFactory class SDFConverter(Converter): @@ -108,7 +33,7 @@ def __init__( self.groupby_key = groupby_key self.n_jobs = n_jobs - def convert(self, path: str, factory: DatasetFactory) -> FactoryProduct: + def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: """ Converts the molecules in an SDF file to a Polaris compatible format. """ @@ -154,6 +79,9 @@ def _get_name(mol: dm.Mol): # This might not be the most storage efficient, but is fastest and easiest to maintain. # We do not save the MolProps, because we have already extracted these into columns. # See: https://github.com/rdkit/rdkit/discussions/7235 + + # NOTE (cwognum): We might want to improve efficiency + # by not always storing private and computed properties. props = Chem.PropertyPickleOptions.AllProps if self.mol_prop_as_cols: props &= ~Chem.PropertyPickleOptions.MolProps @@ -181,7 +109,7 @@ def _get_name(mol: dm.Mol): # Get the pointer path pointer_idx = f"{start}:{end}" if start != end else f"{start}" - pointer = f"{factory.zarr_root_path}/{self.mol_column}#{pointer_idx}" + pointer = self.get_pointer(factory.zarr_root_path, self.mol_column, pointer_idx) # Get the single unique value per column for the group and append unique_values = [group[col].unique()[0] for col in df.columns] @@ -190,7 +118,7 @@ def _get_name(mol: dm.Mol): df = grouped else: - pointers = [f"{factory.zarr_root_path}/{self.mol_column}#{i}" for i in range(len(df))] + pointers = [self.get_pointer(factory.zarr_root_path, self.mol_column, i) for i in range(len(df))] df[self.mol_column] = pd.Series(pointers) # Set the annotations diff --git a/polaris/dataset/converters/_zarr.py b/polaris/dataset/converters/_zarr.py new file mode 100644 index 00000000..c490cfc1 --- /dev/null +++ b/polaris/dataset/converters/_zarr.py @@ -0,0 +1,50 @@ +from collections import defaultdict +from typing import TYPE_CHECKING + +import pandas as pd +import zarr + +from polaris.dataset import ColumnAnnotation +from polaris.dataset.converters._base import Converter, FactoryProduct + +if TYPE_CHECKING: + from polaris.dataset import DatasetFactory + + +class ZarrConverter(Converter): + """Parse a [.zarr](https://zarr.readthedocs.io/en/stable/index.html) hierarchy into a Polaris `Dataset`. + + In short: A `.zarr` file can contain groups and arrays, where each group can again contain groups and arrays. + + Within Polaris: + + 1. Each subgroup of the root group corresponds to a single column. + 2. Each subgroup is in turn expected to contain a single array with _all_ datapoints. + + Tip: Tutorial + To learn more about the zarr format, see the + [tutorial](../tutorials/dataset_zarr.ipynb). + """ + + def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: + src = zarr.open(path, "r") + + v = next(src.group_keys(), None) + if v is not None: + raise ValueError("The root of the zarr hierarchy should only contain arrays.") + + # Construct the table + # Parse any group into a column + data = defaultdict(dict) + for col, arr in src.arrays(): + # Copy to the source zarr, so everything is in one place + dst = zarr.open_group("/".join([factory.zarr_root_path, col]), "w") + zarr.copy(arr, dst) + + for i in range(len(arr)): + # In case all data is saved in a single array, we construct a path with an index suffix. + data[col][i] = self.get_pointer(path, arr.name, i) + + # Construct the dataset + table = pd.DataFrame(data) + return table, {k: ColumnAnnotation(is_pointer=True) for k in table.columns} diff --git a/tests/conftest.py b/tests/conftest.py index cd71fcf2..1105c151 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,21 +12,11 @@ from polaris.utils.types import HubOwner, License -def _get_zarr_archive(tmp_path, datapoint_per_array: bool): +def _get_zarr_archive(tmp_path): tmp_path = fs.join(str(tmp_path), "data.zarr") root = zarr.open_group(tmp_path, mode="w") - group_a = root.create_group("A/") - group_b = root.create_group("B/") - - def _populate_group(group): - if datapoint_per_array: - for i in range(100): - group.array(i, data=np.random.random((2048,))) - else: - group.array("data", data=np.random.random((100, 2048))) - - _populate_group(group_a) - _populate_group(group_b) + root.array("A", data=np.random.random((100, 2048))) + root.array("B", data=np.random.random((100, 2048))) return tmp_path @@ -64,14 +54,9 @@ def test_dataset(test_data, test_org_owner): ) -@pytest.fixture(scope="function") -def test_zarr_archive_multiple_arrays(tmp_path): - return _get_zarr_archive(tmp_path, datapoint_per_array=True) - - @pytest.fixture(scope="function") def test_zarr_archive_single_array(tmp_path): - return _get_zarr_archive(tmp_path, datapoint_per_array=False) + return _get_zarr_archive(tmp_path) @pytest.fixture(scope="function") diff --git a/tests/test_dataset.py b/tests/test_dataset.py index eb1b3965..3ed5ee3f 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -4,7 +4,7 @@ import zarr from pydantic import ValidationError -from polaris.dataset import Dataset +from polaris.dataset import Dataset, get_dataset_from_file from polaris.loader import load_dataset from polaris.utils import fs from polaris.utils.errors import PolarisChecksumError @@ -42,19 +42,23 @@ def test_load_data(tmp_path): tmpdir = str(tmp_path) path = fs.join(tmpdir, "data.zarr") - zarr.save(path, arr) + root = zarr.open(path, "w") + root.array("A", data=arr) + + path = f"{path}/A#0" table = pd.DataFrame({"A": [path]}, index=[0]) - dataset = Dataset(table=table, cache_dir=tmpdir, annotations={"A": {"is_pointer": True}}) + dataset = Dataset(table=table, annotations={"A": {"is_pointer": True}}) # Without caching + data = dataset.get_data(row=0, col="A") - assert (data == arr).all() + assert (data == arr[0]).all() # With caching - dataset.cache() + dataset.cache(tmpdir) data = dataset.get_data(row=0, col="A") - assert (data == arr).all() + assert (data == arr[0]).all() def test_dataset_checksum(test_dataset): @@ -99,6 +103,17 @@ def _check_for_failure(_kwargs): assert dataset.md5sum is not None +def test_dataset_from_zarr(test_zarr_archive_single_array, tmpdir): + """Test whether loading works when the zarr archive contains a single array or multiple arrays.""" + archive = test_zarr_archive_single_array + dataset = get_dataset_from_file(archive, tmpdir.join("data")) + + assert len(dataset.table) == 100 + for i in range(100): + assert dataset.get_data(row=i, col="A").shape == (2048,) + assert dataset.get_data(row=i, col="B").shape == (2048,) + + def test_dataset_from_json(test_dataset, tmpdir): """Test whether the dataset can be saved and loaded from json.""" test_dataset.to_json(str(tmpdir)) @@ -112,27 +127,37 @@ def test_dataset_from_json(test_dataset, tmpdir): assert _equality_test(test_dataset, new_dataset) -@pytest.mark.parametrize("array_per_datapoint", [True, False]) -def test_dataset_caching( - test_zarr_archive_single_array, - test_zarr_archive_multiple_arrays, - array_per_datapoint, - tmpdir, -): +def test_dataset_from_zarr_to_json_and_back(test_zarr_archive_single_array, tmpdir): + """ + Test whether a dataset with pointer columns, instantiated from a zarr archive, + can be saved to and loaded from json. + """ + + json_dir = tmpdir.join("json") + zarr_dir = tmpdir.join("zarr") + + archive = test_zarr_archive_single_array + dataset = get_dataset_from_file(archive, zarr_dir) + path = dataset.to_json(json_dir) + + new_dataset = Dataset.from_json(path) + assert _equality_test(dataset, new_dataset) + + new_dataset = load_dataset(path) + assert _equality_test(dataset, new_dataset) + + +def test_dataset_caching(test_zarr_archive_single_array, tmpdir): """Test whether the dataset remains the same after caching.""" - archive = test_zarr_archive_multiple_arrays if array_per_datapoint else test_zarr_archive_single_array + archive = test_zarr_archive_single_array - original_dataset = Dataset.from_zarr(archive) - cached_dataset = Dataset.from_zarr(archive) + original_dataset = get_dataset_from_file(archive, tmpdir.join("original1")) + cached_dataset = get_dataset_from_file(archive, tmpdir.join("original2")) assert original_dataset == cached_dataset + cache_dir = cached_dataset.cache(tmpdir.join("cached").strpath) for i in range(len(cached_dataset)): - assert not cached_dataset.table.loc[i, "A"].startswith(original_dataset.cache_dir) - assert not cached_dataset.table.loc[i, "B"].startswith(original_dataset.cache_dir) - - cached_dataset.cache() - for i in range(len(cached_dataset)): - assert cached_dataset.table.loc[i, "A"].startswith(original_dataset.cache_dir) - assert cached_dataset.table.loc[i, "B"].startswith(original_dataset.cache_dir) + assert cached_dataset.table.loc[i, "A"].startswith(cache_dir) + assert cached_dataset.table.loc[i, "B"].startswith(cache_dir) assert _equality_test(cached_dataset, original_dataset) diff --git a/tests/test_subset.py b/tests/test_subset.py index c6e1c4c5..b00ef865 100644 --- a/tests/test_subset.py +++ b/tests/test_subset.py @@ -85,42 +85,3 @@ def test_input_featurization(test_single_task_benchmark): x = test.X[0] assert isinstance(x, np.ndarray) - - -@pytest.mark.parametrize("fmt", ["dict", "tuple"]) -def test_different_subset_formats_single_task(test_single_task_benchmark, fmt): - train, _ = test_single_task_benchmark.get_train_test_split(target_format=fmt) - assert isinstance(train.y, np.ndarray) - assert train.y.shape == (len(train),) - assert isinstance(train[0][1], float) - assert isinstance(next(train)[1], float) - - -def test_different_subset_formats_multi_task_dict(test_multi_task_benchmark): - train, _ = test_multi_task_benchmark.get_train_test_split(target_format="dict") - assert isinstance(train.y, dict) - assert all(c in test_multi_task_benchmark.target_cols for c in train.y) - assert all(isinstance(v, np.ndarray) and v.shape == (len(train),) for v in train.y.values()) - assert isinstance(train[0][1], dict) - assert isinstance(next(train)[1], dict) - - -def test_different_subset_formats_multi_task_tuple(test_multi_task_benchmark): - train, _ = test_multi_task_benchmark.get_train_test_split(target_format="tuple") - assert isinstance(train.y, np.ndarray) - assert train.y.shape == (len(train), len(train.target_cols)) - assert isinstance(train[0][1], tuple) - assert isinstance(next(train)[1], tuple) - - -def test_consistency_between_different_formats(test_multi_task_benchmark): - train_tup, _ = test_multi_task_benchmark.get_train_test_split(target_format="tuple") - train_dict, _ = test_multi_task_benchmark.get_train_test_split(target_format="dict") - - t = train_tup[0][1] - d = train_dict[0][1] - - assert len(d) == len(t) - for k, v in d.items(): - idx = test_multi_task_benchmark.target_cols.index(k) - assert t[idx] == v diff --git a/tests/test_to_zarr_converters.py b/tests/test_to_zarr_converters.py index 4953e9e9..84f11895 100644 --- a/tests/test_to_zarr_converters.py +++ b/tests/test_to_zarr_converters.py @@ -2,6 +2,7 @@ def test_sdf_bytestring_compat(tmpdir): + "CCC(=O)F", "CC=C(O)F" dm.Mol print(tmpdir) pass From b4fdea8e06ac96c2dee95400771c8a7a5134e499 Mon Sep 17 00:00:00 2001 From: cwognum Date: Wed, 13 Mar 2024 15:10:26 -0400 Subject: [PATCH 3/9] Make it possible to merge dataframes --- polaris/dataset/_factory.py | 28 ++++++++++++++++++++++++---- polaris/dataset/converters/_base.py | 2 +- polaris/dataset/converters/_sdf.py | 5 +++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py index 5a6a5124..85a02d13 100644 --- a/polaris/dataset/_factory.py +++ b/polaris/dataset/_factory.py @@ -64,6 +64,8 @@ def add_column( """Adds a single column""" if column.name is None: raise RuntimeError("You need to specify a column name") + if column.name in self.table.columns: + raise ValueError(f"Column name '{column.name}' already exists in the table") if annotation is not None and annotation.is_pointer: if self.zarr_root is None: @@ -75,17 +77,35 @@ def add_column( annotation = ColumnAnnotation() self.annotations[column.name] = annotation + def add_columns( + self, + df: pd.DataFrame, + annotations: Optional[Dict[str, ColumnAnnotation]] = None, + merge_on: Optional[str] = None, + ): + """Adds a single column""" + if merge_on is not None: + df = self.table.merge(df, on=merge_on, how="outer") + + if annotations is None: + annotations = {} + annotations = {**self.annotations, **annotations} + + if merge_on is not None: + self.reset() + + for name, series in df.items(): + annotation = annotations.get(name) + self.add_column(series, annotation) + def add_from_file(self, path: str): - """ """ ext = dm.fs.get_extension(path) converter = self._converters.get(ext) if converter is None: raise ValueError(f"No converter found for extension {ext}") table, annotations = converter.convert(path, self) - - for name, series in table.items(): - self.add_column(series, annotations.get(name)) + self.add_columns(table, annotations) def build(self) -> Dataset: return Dataset(table=self.table, annotations=self.annotations) diff --git a/polaris/dataset/converters/_base.py b/polaris/dataset/converters/_base.py index cc87d8b2..b51ceec5 100644 --- a/polaris/dataset/converters/_base.py +++ b/polaris/dataset/converters/_base.py @@ -29,4 +29,4 @@ def get_pointer(root: str, column: str, index: Union[int, slice]) -> str: index_substr = f"{_INDEX_SEP}{index.start}:{index.stop}" else: index_substr = f"{_INDEX_SEP}{index}" - return f"{root}{column}{index_substr}" + return f"{root}/{column}{index_substr}" diff --git a/polaris/dataset/converters/_sdf.py b/polaris/dataset/converters/_sdf.py index ca5eb80f..bc773650 100644 --- a/polaris/dataset/converters/_sdf.py +++ b/polaris/dataset/converters/_sdf.py @@ -49,7 +49,6 @@ def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: mol_column=tmp_col, remove_hs=False, sanitize=False, - max_num_mols=1000, ) if not isinstance(df, pd.DataFrame): @@ -72,7 +71,9 @@ def _get_name(mol: dm.Mol): # Add a column with the SMILES if it doesn't exist yet if self.smiles_column is not None and self.smiles_column not in df.columns: - names = dm.parallelized(dm.to_smiles, df[tmp_col], n_jobs=self.n_jobs) + names = dm.parallelized( + lambda mol: dm.to_smiles(mol, isomeric=False), df[tmp_col], n_jobs=self.n_jobs + ) df[self.smiles_column] = names # Convert the molecules to binary strings (for ML purposes, this should be lossless). From 5f943756f057a941b329b1e88b349f9c65ea95be Mon Sep 17 00:00:00 2001 From: cwognum Date: Wed, 13 Mar 2024 18:23:49 -0400 Subject: [PATCH 4/9] Added docstrings --- docs/api/adapters.md | 20 ++++ docs/api/converters.md | 18 ++++ docs/api/factory.md | 11 ++ mkdocs.yml | 5 +- polaris/dataset/__init__.py | 1 - polaris/dataset/_adapters.py | 40 +++++++- polaris/dataset/_factory.py | 149 ++++++++++++++++++++++------ polaris/dataset/converters/_sdf.py | 46 +++++---- polaris/dataset/converters/_zarr.py | 19 ++-- polaris/loader/load.py | 22 ++-- 10 files changed, 258 insertions(+), 73 deletions(-) create mode 100644 docs/api/adapters.md create mode 100644 docs/api/converters.md create mode 100644 docs/api/factory.md diff --git a/docs/api/adapters.md b/docs/api/adapters.md new file mode 100644 index 00000000..5eb189e5 --- /dev/null +++ b/docs/api/adapters.md @@ -0,0 +1,20 @@ + +## Base Class + +::: polaris.dataset._adapters.Adapter + options: + filters: ["!^__init__"] + +--- + +## Implementations + +::: polaris.dataset._adapters.SmilesAdapter + options: + filters: ["!^__init__"] + +::: polaris.dataset._adapters.MolBytestringAdapter + options: + filters: ["!^__init__"] + +--- \ No newline at end of file diff --git a/docs/api/converters.md b/docs/api/converters.md new file mode 100644 index 00000000..9be10eed --- /dev/null +++ b/docs/api/converters.md @@ -0,0 +1,18 @@ +::: polaris.dataset.converters.Converter + options: + filters: ["!^_"] + +--- + + +::: polaris.dataset.converters.SDFConverter + options: + filters: ["!^_"] + +--- + +::: polaris.dataset.converters.ZarrConverter + options: + filters: ["!^_"] + +--- diff --git a/docs/api/factory.md b/docs/api/factory.md new file mode 100644 index 00000000..3e076a7d --- /dev/null +++ b/docs/api/factory.md @@ -0,0 +1,11 @@ +::: polaris.dataset.DatasetFactory + options: + filters: ["!^_"] + +--- + +::: polaris.dataset.get_dataset_from_file + options: + filters: ["!^_"] + +--- \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 56e3785a..28dc8e8e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,7 +20,7 @@ nav: - The Basics: tutorials/basics.ipynb - Data Curation: tutorials/data_curation.ipynb - Custom Datasets and Benchmarks: tutorials/custom_dataset_benchmark.ipynb - # - Creating Datasets with zarr: tutorials/dataset_zarr.ipynb + - Creating Datasets with zarr: tutorials/dataset_zarr.ipynb - API Reference: - Load: api/load.md - Core: @@ -32,6 +32,9 @@ nav: - Client: api/hub.client.md - PolarisFileSystem: api/hub.polarisfs.md - Additional: + - Dataset Factory: api/factory.md + - Data Converters: api/converters.md + - Data Adapters: api/adapters.md - Base classes: api/base.md - Types: api/utils.types.md - Community: community/community.md diff --git a/polaris/dataset/__init__.py b/polaris/dataset/__init__.py index 198b28f6..b09404d9 100644 --- a/polaris/dataset/__init__.py +++ b/polaris/dataset/__init__.py @@ -1,4 +1,3 @@ -from polaris.dataset._adapters import Adapter from polaris.dataset._column import ColumnAnnotation, Modality from polaris.dataset._dataset import Dataset from polaris.dataset._factory import DatasetFactory, get_dataset_from_file diff --git a/polaris/dataset/_adapters.py b/polaris/dataset/_adapters.py index 6e9311ff..0d5a1008 100644 --- a/polaris/dataset/_adapters.py +++ b/polaris/dataset/_adapters.py @@ -1,13 +1,30 @@ import abc +from typing import Any import datamol as dm from pydantic import BaseModel class Adapter(BaseModel, abc.ABC): + """ + Adapters are callable, serializable objects that can be used to _adapt_ the + datapoint in a dataset. This is for example + """ + column: str def __call__(self, data: dict) -> dict: + """Adapts the entire datapoint + + Used like: + ```python + adapter = Adapter(column="my_column") + adapter({"my_column": datapoint}) + ``` + + Args: + data: The entire datapoint with column -> value pairs. + """ if self.column not in data: return data v = data[self.column] @@ -18,15 +35,36 @@ def __call__(self, data: dict) -> dict: return data @abc.abstractmethod - def adapt(self, data: dict): + def adapt(self, data: Any) -> Any: + """ + Adapt the value for a specific column. + This method has to be overwritten by subclasses. + + Used like: + ```python + adapter = Adapter(column="my_column") + adapter().adapt(datapoint["my_column"]) + ``` + + Args: + data: The value to adapt + """ raise NotImplementedError class SmilesAdapter(Adapter): + """ + Creates a RDKit `Mol` object from a SMILES string + """ + def adapt(self, data: str) -> dm.Mol: return dm.to_mol(data) class MolBytestringAdapter(Adapter): + """ + Creates a RDKit `Mol` object from the RDKit-specific bytestring serialization + """ + def adapt(self, data: bytes) -> dm.Mol: return dm.Mol(data) diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py index 85a02d13..ae90e14b 100644 --- a/polaris/dataset/_factory.py +++ b/polaris/dataset/_factory.py @@ -4,16 +4,18 @@ import datamol as dm import pandas as pd import zarr +from loguru import logger from polaris.dataset import ColumnAnnotation, Dataset -from polaris.dataset.converters import SDFConverter, ZarrConverter +from polaris.dataset.converters import Converter, SDFConverter, ZarrConverter def get_dataset_from_file(path: str, zarr_root_path: Optional[str] = None) -> Dataset: """ This function is a convenience function to create a dataset from a file. - It uses the factory design pattern to create the dataset. - For more complicated datasets, please use the `DatasetFactory` directly. + + It sets up the dataset factory with sensible defaults for the converters. + For creating more complicated datasets, please use the `DatasetFactory` directly. """ factory = DatasetFactory(zarr_root_path=zarr_root_path) factory.register_converter("sdf", SDFConverter()) @@ -25,57 +27,108 @@ def get_dataset_from_file(path: str, zarr_root_path: Optional[str] = None) -> Da class DatasetFactory: """ - The DatasetFactory is meant to more easily create complex datasets. - It uses the factory design pattern. + The `DatasetFactory` makes it easier to create complex datasets. + + It is based on the the factory design pattern and allows a user to specify specific handlers + (i.e. [`Converter`][polaris.dataset.converters._base.Converter] objects) for different file types. + These converters are used to convert commonly used file types in drug discovery + to something that can be used within Polaris while losing as little information as possible. + + In addition, it contains utility method to incrementally build out a dataset from different sources. + + Tip: Try quickly converting one of your datasets + The `DatasetFactory` is designed to give you full control. + If your dataset is saved in a single file and you don't need anything fancy, you can try use + [`get_dataset_from_file`][polaris.dataset.get_dataset_from_file] instead. + + ```py + from polaris.dataset import get_dataset_from_file + dataset = get_dataset_from_file("path/to/my_dataset.sdf") + ``` + + Question: How to make adding meta-data easier? + The `DatasetFactory` is designed to more easily pull together data from different sources. + However, adding meta-data remains a laborous process. How could we make this simpler through + the Python API? """ def __init__(self, zarr_root_path: Optional[str] = None) -> None: - self.zarr_root_path = os.path.abspath(zarr_root_path).rstrip("/") + """ + Create a new factory object. + + Args: + zarr_root_path: The root path of the zarr hierarchy. If you want to use pointer columns, + this arguments needs to be passed. + """ + self._zarr_root_path = os.path.abspath(zarr_root_path).rstrip("/") self._zarr_root = None - - self.table: pd.DataFrame = pd.DataFrame() - self.annotations: Dict[str, ColumnAnnotation] = {} - + self._table: pd.DataFrame = pd.DataFrame() + self._annotations: Dict[str, ColumnAnnotation] = {} self._converters = {} @property def zarr_root(self) -> zarr.Group: - if self.zarr_root_path is None: + """ + The root of the zarr archive for the Dataset that is being built. + All data for a single dataset is expected to be stored in the same Zarr archive. + """ + if self._zarr_root_path is None: raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") if self._zarr_root is None: - self._zarr_root = zarr.open(self.zarr_root_path, "w") + self._zarr_root = zarr.open(self._zarr_root_path, "w") if not isinstance(self._zarr_root, zarr.Group): raise ValueError("The root of the zarr hierarchy should be a group") return self._zarr_root - def register_converter(self, ext: str, converter): + def register_converter(self, ext: str, converter: Converter): + """ + Registers a new converter for a specific file type. + + Args: + ext: The file extension for which the converter should be used. + There can only be a single converter per file extension. + converter: The handler for the file type. This should convert + the file to a Polaris-compatible format. + """ + if ext in self._converters: + logger.info(f"You are overwriting the converter for the {ext} extension.") self._converters[ext] = converter - def reset(self): - self.table = pd.DataFrame() - self.annotations = {} + def add_column(self, column: pd.Series, annotation: Optional[ColumnAnnotation] = None): + """ + Add a single column to the DataFrame - def add_column( - self, - column: pd.Series, - annotation: Optional[ColumnAnnotation] = None, - ): - """Adds a single column""" + We require: + + 1. The name attribute of the column to be set. + 2. The name attribute of the column to be unique. + 3. If the column is a pointer column, the `zarr_root_path` needs to be set. + 4. The length of the column to match the length of the alredy constructed table. + + Args: + column: The column to add to the dataset. + annotation: The annotation for the column. If None, a default annotation will be used. + """ + + # Verify the column can be added if column.name is None: - raise RuntimeError("You need to specify a column name") - if column.name in self.table.columns: + raise ValueError("You need to specify a column name") + if column.name in self._table.columns: raise ValueError(f"Column name '{column.name}' already exists in the table") + if not self._table.empty and len(column) != len(self._table): + raise ValueError("The length of the column does not match the length of the table") if annotation is not None and annotation.is_pointer: if self.zarr_root is None: raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") - self.table[column.name] = column + # Actually add the column + self._table[column.name] = column if annotation is None: annotation = ColumnAnnotation() - self.annotations[column.name] = annotation + self._annotations[column.name] = annotation def add_columns( self, @@ -83,13 +136,28 @@ def add_columns( annotations: Optional[Dict[str, ColumnAnnotation]] = None, merge_on: Optional[str] = None, ): - """Adds a single column""" + """ + Add multiple columns to the dataset based on another dataframe. + + To have more control over how the two dataframes are combined, you can + specify a column to merge on. This will always do an **outer** join. + + If not specifying a key to merge on, the columns will simply be added to the dataset + that has been built so far without any reordering. They are therefore expected to meet all + the same expectations as for [`add_column`][polaris.dataset.DatasetFactory.add_column]. + + Args: + df: A Pandas DataFrame with the columns that we want to add to the dataset. + annotations: The annotations for the columns. If None, default annotations will be used. + merge_on: The column to merge on, if any. + """ + if merge_on is not None: - df = self.table.merge(df, on=merge_on, how="outer") + df = self._table.merge(df, on=merge_on, how="outer") if annotations is None: annotations = {} - annotations = {**self.annotations, **annotations} + annotations = {**self._annotations, **annotations} if merge_on is not None: self.reset() @@ -99,6 +167,13 @@ def add_columns( self.add_column(series, annotation) def add_from_file(self, path: str): + """ + Uses the registered converters to parse the data from a specific file and add it to the dataset. + If no converter is found for the file extension, it raises an error. + + Args: + path: The path to the file that should be parsed. + """ ext = dm.fs.get_extension(path) converter = self._converters.get(ext) if converter is None: @@ -108,4 +183,18 @@ def add_from_file(self, path: str): self.add_columns(table, annotations) def build(self) -> Dataset: - return Dataset(table=self.table, annotations=self.annotations) + """Returns a Dataset based on the current state of the factory.""" + return Dataset(table=self._table, annotations=self._annotations) + + def reset(self, zarr_root_path: Optional[str] = None): + """ + Resets the factory to its initial state to start building the next dataset from scratch. + Note that this will not reset the registered converters. + + Args: + zarr_root_path: The root path of the zarr hierarchy. If you want to use pointer columns + for your next dataset, this arguments needs to be passed. + """ + self._zarr_root_path = zarr_root_path + self._table = pd.DataFrame() + self._annotations = {} diff --git a/polaris/dataset/converters/_sdf.py b/polaris/dataset/converters/_sdf.py index bc773650..f7339147 100644 --- a/polaris/dataset/converters/_sdf.py +++ b/polaris/dataset/converters/_sdf.py @@ -13,31 +13,47 @@ class SDFConverter(Converter): - """Convert from a SDF file""" + """ + Converts a SDF file into a Polaris dataset. + + Info: Binary strings for serialization + This class converts the molecules to binary strings (for ML purposes, this should be lossless). + This might not be the most storage efficient, but is fastest and easiest to maintain. + See this [Github Discussion](https://github.com/rdkit/rdkit/discussions/7235) for more info. + + Properties defined on the molecule level in the SDF file can be extracted into separate columns + or can be kept in the molecule object. + + Args: + mol_column: The name of the column that will contain the pointers to the molecules. + smiles_column: The name of the column that will contain the SMILES strings. + use_isomeric_smiles: Whether to use isomeric SMILES. + mol_id_column: The name of the column that will contain the molecule names. + mol_prop_as_cols: Whether to extract properties defined on the molecule level in the SDF file into separate columns. + groupby_key: The name of the column to group by. If set, the dataset can combine multiple pointers + to the molecules into a single datapoint. + """ def __init__( self, mol_column: str = "molecule", smiles_column: Optional[str] = "smiles", + use_isomeric_smiles: bool = True, mol_id_column: Optional[str] = None, mol_prop_as_cols: bool = True, groupby_key: Optional[str] = None, n_jobs: int = 1, ) -> None: - """ """ super().__init__() self.mol_column = mol_column self.smiles_column = smiles_column + self.use_isomeric_smiles = use_isomeric_smiles self.mol_id_column = mol_id_column self.mol_prop_as_cols = mol_prop_as_cols self.groupby_key = groupby_key self.n_jobs = n_jobs def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: - """ - Converts the molecules in an SDF file to a Polaris compatible format. - """ - tmp_col = uuid.uuid4().hex # We do not sanitize the molecules or remove the Hs. @@ -72,17 +88,14 @@ def _get_name(mol: dm.Mol): # Add a column with the SMILES if it doesn't exist yet if self.smiles_column is not None and self.smiles_column not in df.columns: names = dm.parallelized( - lambda mol: dm.to_smiles(mol, isomeric=False), df[tmp_col], n_jobs=self.n_jobs + lambda mol: dm.to_smiles(mol, isomeric=self.use_isomeric_smiles), + df[tmp_col], + n_jobs=self.n_jobs, ) df[self.smiles_column] = names - # Convert the molecules to binary strings (for ML purposes, this should be lossless). - # This might not be the most storage efficient, but is fastest and easiest to maintain. - # We do not save the MolProps, because we have already extracted these into columns. - # See: https://github.com/rdkit/rdkit/discussions/7235 - - # NOTE (cwognum): We might want to improve efficiency - # by not always storing private and computed properties. + # Convert the molecules to binary strings. This should be lossless and efficient. + # NOTE (cwognum): We might want to not always store private and computed properties. props = Chem.PropertyPickleOptions.AllProps if self.mol_prop_as_cols: props &= ~Chem.PropertyPickleOptions.MolProps @@ -95,7 +108,6 @@ def _get_name(mol: dm.Mol): # Add a pointer column to the table # We support grouping by a key, to allow inputs of variable length - grouped = pd.DataFrame(columns=[*df.columns, self.mol_column]) if self.groupby_key is not None: for _, group in df.reset_index(drop=True).groupby(by=self.groupby_key): @@ -110,7 +122,7 @@ def _get_name(mol: dm.Mol): # Get the pointer path pointer_idx = f"{start}:{end}" if start != end else f"{start}" - pointer = self.get_pointer(factory.zarr_root_path, self.mol_column, pointer_idx) + pointer = self.get_pointer(factory._zarr_root_path, self.mol_column, pointer_idx) # Get the single unique value per column for the group and append unique_values = [group[col].unique()[0] for col in df.columns] @@ -119,7 +131,7 @@ def _get_name(mol: dm.Mol): df = grouped else: - pointers = [self.get_pointer(factory.zarr_root_path, self.mol_column, i) for i in range(len(df))] + pointers = [self.get_pointer(factory._zarr_root_path, self.mol_column, i) for i in range(len(df))] df[self.mol_column] = pd.Series(pointers) # Set the annotations diff --git a/polaris/dataset/converters/_zarr.py b/polaris/dataset/converters/_zarr.py index c490cfc1..def5d906 100644 --- a/polaris/dataset/converters/_zarr.py +++ b/polaris/dataset/converters/_zarr.py @@ -12,18 +12,19 @@ class ZarrConverter(Converter): - """Parse a [.zarr](https://zarr.readthedocs.io/en/stable/index.html) hierarchy into a Polaris `Dataset`. - - In short: A `.zarr` file can contain groups and arrays, where each group can again contain groups and arrays. - - Within Polaris: - - 1. Each subgroup of the root group corresponds to a single column. - 2. Each subgroup is in turn expected to contain a single array with _all_ datapoints. + """Parse a [.zarr](https://zarr.readthedocs.io/en/stable/index.html) archive into a Polaris `Dataset`. Tip: Tutorial To learn more about the zarr format, see the [tutorial](../tutorials/dataset_zarr.ipynb). + + Warning: Loading from `.zarr` + Loading and saving datasets from and to `.zarr` is still experimental and currently not + fully supported by the Hub. + + A `.zarr` file can contain groups and arrays, where each group can again contain groups and arrays. + Within Polaris, the Zarr archive is expected to have a flat hierarchy where each array corresponds + to a single column and each array contains the values for all datapoints in that column. """ def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: @@ -38,7 +39,7 @@ def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: data = defaultdict(dict) for col, arr in src.arrays(): # Copy to the source zarr, so everything is in one place - dst = zarr.open_group("/".join([factory.zarr_root_path, col]), "w") + dst = zarr.open_group("/".join([factory._zarr_root_path, col]), "w") zarr.copy(arr, dst) for i in range(len(arr)): diff --git a/polaris/loader/load.py b/polaris/loader/load.py index b6a3cc1a..0472908d 100644 --- a/polaris/loader/load.py +++ b/polaris/loader/load.py @@ -1,11 +1,12 @@ +import json + import fsspec -import yaml from polaris.benchmark._definitions import ( MultiTaskBenchmarkSpecification, SingleTaskBenchmarkSpecification, ) -from polaris.dataset._dataset import Dataset +from polaris.dataset import Dataset, get_dataset_from_file from polaris.hub.client import PolarisHubClient from polaris.utils import fs @@ -24,12 +25,8 @@ def load_dataset(path: str) -> Dataset: provide the `owner/name` slug. This can be easily copied from the relevant dataset page on the Hub. - **Directory**: When loading the dataset from a directory, you should provide the path - as returned by [`Dataset.to_json`][polaris.dataset.Dataset.to_json] or - [`Dataset.to_zarr`][polaris.dataset.Dataset.to_zarr]. The path can be local or remote. - - Warning: Loading from `.zarr` - Loading and saving datasets from and to `.zarr` is still experimental and currently not - supported by the Hub. + as returned by [`Dataset.to_json`][polaris.dataset.Dataset.to_json]. + The path can be local or remote. """ extension = fs.get_extension(path) @@ -40,12 +37,9 @@ def load_dataset(path: str) -> Dataset: client = PolarisHubClient() return client.get_dataset(*path.split("/")) - if extension == "zarr": - return Dataset.from_zarr(path) - elif extension == "json": + if extension == "json": return Dataset.from_json(path) - - raise NotImplementedError("This should not be reached.") + return get_dataset_from_file(path) def load_benchmark(path: str): @@ -75,7 +69,7 @@ def load_benchmark(path: str): return client.get_benchmark(*path.split("/")) with fsspec.open(path, "r") as fd: - data = yaml.safe_load(fd) # type: ignore + data = json.load(fd) # TODO (cwognum): As this gets more complex, how do we effectivly choose which class we should use? # e.g. we might end up with a single class per benchmark. From 9cfbebaa170665a22ac31617494d02a2b19279d6 Mon Sep 17 00:00:00 2001 From: cwognum Date: Thu, 14 Mar 2024 15:31:34 -0400 Subject: [PATCH 5/9] Updated docs and tutorials --- docs/api/adapters.md | 20 +- docs/api/factory.md | 2 +- docs/tutorials/basics.ipynb | 17 +- docs/tutorials/custom_dataset_benchmark.ipynb | 13 +- docs/tutorials/data_curation.ipynb | 4 +- docs/tutorials/dataset_factory.ipynb | 589 ++++++++++++++++++ docs/tutorials/dataset_zarr.ipynb | 481 +++++++------- mkdocs.yml | 10 +- polaris/dataset/__init__.py | 5 +- polaris/dataset/_adapters.py | 74 +-- polaris/dataset/_dataset.py | 66 +- polaris/dataset/_factory.py | 60 +- polaris/dataset/_subset.py | 12 +- polaris/dataset/converters/_base.py | 3 +- polaris/dataset/converters/_sdf.py | 7 +- polaris/dataset/converters/_zarr.py | 4 +- polaris/loader/load.py | 4 +- tests/test_dataset.py | 10 +- tests/test_integration.py | 1 - 19 files changed, 1021 insertions(+), 361 deletions(-) create mode 100644 docs/tutorials/dataset_factory.ipynb diff --git a/docs/api/adapters.md b/docs/api/adapters.md index 5eb189e5..21de9d95 100644 --- a/docs/api/adapters.md +++ b/docs/api/adapters.md @@ -1,20 +1,4 @@ -## Base Class - -::: polaris.dataset._adapters.Adapter - options: - filters: ["!^__init__"] - ---- - -## Implementations - -::: polaris.dataset._adapters.SmilesAdapter - options: - filters: ["!^__init__"] - -::: polaris.dataset._adapters.MolBytestringAdapter +::: polaris.dataset._adapters options: - filters: ["!^__init__"] - ---- \ No newline at end of file + filters: ["!^_"] diff --git a/docs/api/factory.md b/docs/api/factory.md index 3e076a7d..6ede77d6 100644 --- a/docs/api/factory.md +++ b/docs/api/factory.md @@ -4,7 +4,7 @@ --- -::: polaris.dataset.get_dataset_from_file +::: polaris.dataset.create_dataset_from_file options: filters: ["!^_"] diff --git a/docs/tutorials/basics.ipynb b/docs/tutorials/basics.ipynb index 22bbb0f0..50bc2802 100644 --- a/docs/tutorials/basics.ipynb +++ b/docs/tutorials/basics.ipynb @@ -3,7 +3,13 @@ { "cell_type": "markdown", "id": "40f99374-b47e-4f84-bdb9-148a11f9c07d", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "# The Basics\n", "\n", @@ -12,7 +18,7 @@ "

This tutorial walks you through the basic usage of Polaris. We will first login to the hub and will then see how easy it is to load a dataset or benchmark from it. Finally, we will train a simple baseline to submit a first set of results!

\n", "\n", "\n", - "Polaris is designed to standardize the process of constructing datasets, specifying benchmarks and evaluating novel machine learning techniques within the realms of biology, chemistry, and drug discovery.\n", + "Polaris is designed to standardize the process of constructing datasets, specifying benchmarks and evaluating novel machine learning techniques within the realm of drug discovery.\n", "\n", "While the Polaris library can be used independently from the Polaris Hub, the two were designed to seamlessly work together. The hub provides various pre-made, high quality datasets and benchmarks to develop and evaluate novel ML methods. In this tutorial, we will see how easy it is to load and use these datasets and benchmarks." ] @@ -22,6 +28,10 @@ "execution_count": 1, "id": "3d66f466", "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, "tags": [ "remove_cell" ] @@ -761,7 +771,8 @@ "metadata": {}, "outputs": [], "source": [ - "results.name = f\"hello-world-result\"\n", + "# For a complete list of meta-data, check out the BenchmarkResults object\n", + "results.name = \"hello-world-result\"\n", "results.github_url = \"https://github.com/polaris-hub/polaris-hub\"\n", "results.paper_url = \"https://polarishub.io/\"\n", "results.description = \"Hello, World!\"" diff --git a/docs/tutorials/custom_dataset_benchmark.ipynb b/docs/tutorials/custom_dataset_benchmark.ipynb index 5d1ca9ab..37c031ad 100644 --- a/docs/tutorials/custom_dataset_benchmark.ipynb +++ b/docs/tutorials/custom_dataset_benchmark.ipynb @@ -3,15 +3,20 @@ { "cell_type": "markdown", "id": "172ae3e5", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ - "# Custom datasets and benchmarks\n", "
\n", "

In short

\n", "

This tutorial walks you through the dataset and benchmark data-structures. After creating our own custom dataset and benchmark, we will learn how to upload it to the Hub!

\n", "
\n", "\n", - "We have already seen how easy it is to load a benchmark or dataset from the Polaris Hub. Let's now see how you could create your own!" + "We have already seen how easy it is to load a benchmark or dataset from the Polaris Hub. Let's now learn a bit more about the underlying data model by creating our own dataset and benchmark!" ] }, { @@ -626,7 +631,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/docs/tutorials/data_curation.ipynb b/docs/tutorials/data_curation.ipynb index 9502f97c..270a8a47 100644 --- a/docs/tutorials/data_curation.ipynb +++ b/docs/tutorials/data_curation.ipynb @@ -5,8 +5,6 @@ "id": "40f99374-b47e-4f84-bdb9-148a11f9c07d", "metadata": {}, "source": [ - "# Dataset curation\n", - "\n", "
\n", "

In short

\n", "

This tutorial shows how we can use the curation API of the Polaris library to check and improve the quality of a dataset.

\n", @@ -3767,7 +3765,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/docs/tutorials/dataset_factory.ipynb b/docs/tutorials/dataset_factory.ipynb new file mode 100644 index 00000000..28220c2f --- /dev/null +++ b/docs/tutorials/dataset_factory.ipynb @@ -0,0 +1,589 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e558d600-68d2-473f-89b4-4a356277c078", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "remove_cell" + ] + }, + "outputs": [], + "source": [ + "# Note: Cell is tagged to not show up in the mkdocs build\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "id": "f842d55c-6327-4e81-ba07-79eafe9d47a3", + "metadata": {}, + "source": [ + "
\n", + "

In short

\n", + "

This tutorial shows how we can create more complicated datasets by leveraging the dataset factory in Polaris.

\n", + "
\n", + "\n", + "
\n", + "

This feature is still very new

\n", + "

The features we will show in this tutorial are still experimental. We would love to learn from the community how we can make it easier to create datasets.

\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "278ce19e-0b47-43f1-9876-b3b69a2154e1", + "metadata": {}, + "outputs": [], + "source": [ + "import platformdirs\n", + "import datamol as dm" + ] + }, + { + "cell_type": "markdown", + "id": "856afc27-6f9e-40a1-97c2-e4c1188b2faf", + "metadata": {}, + "source": [ + "## Dataset Factory\n", + "Datasets in Polaris are expected to be saved in a very specific format. This format has been carefully designed to be as universal and performant as possible. Nevertheless, we expect very few datasets to be readily available in this format. We therefore provide the `DatasetFactory` as a way to more easily convert datasets to the Polaris specific format.\n", + "\n", + "Let's assume we have a dataset in the SDF format. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d8b3087e-8c50-45b4-ada7-44bf783cc929", + "metadata": {}, + "outputs": [], + "source": [ + "SAVE_DIR = dm.fs.join(platformdirs.user_cache_dir(appname=\"polaris-tutorials\"), \"003\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "0776f067-d01b-4b7c-89f6-a3c817f934fb", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZJUlEQVR4nO3daVRUV7428KcoJsEogzKoqEGNA6IROlmixAEFkTAognMwU5t4jencmNtqnAhxwE7U+LbRqFfTGLEJgwSnqKgdlwja7dAK6DXIKCJDREQJFFXUeT8U7YCaCHUGlef3SXdV7f2vtWo97HPOPvuoBEEAERG1lInSBRARPdsYo0RERmGMEhEZhTFKRGQUxigRkVEYo0RERjFVugAi2Wm1OHQIly5BENC3L/z8YG6udE30DFNx3Si1Ljk5CAxERQX+8AcAOH0a9vbYvRt9+ypdGT2rGKPUmuh0GDgQtrbYvx/t2gFAdTUCAlBejuxsmJkpXR89k3hulFqTAwdw8SLWrm3MUADt2mHdOuTkYO9eRSujZxhjlFqTjAy0b49XXnmg0dMT9vbIyFCoJnrmMUapNamshLPzI9o7d8aNG7JXQ88Jxii1JlZWqKx8RPuNG2jbVvZq6DnBGKXWxN0dFRVNJ55lZbh+He7uCtVEzzzGKLUmISFo2xbR0Q80rlgBa2uEhipUEz3zuPyeWhNbW2zZgunTUVgIX1+oVEhNxa5d2L4ddnZKF0fPKq4bpdbh+HFoNBg9GgDOnMHXXyMrC4KA/v0xe3bjUnyiFmGMUisgCPD0xLlziIvDpElKV0PPG54bpVYgORnnzsHZGcHBAKDVIi1N6Zro+cEYpeedICAqCgAWLUKbNgAQE4PXXsOsWcrWRc8NxigpQxCE2tpaOUZKTMT583BxwTvvAIBWixUrAGDECDlGp1aAMUoK2LNnj6ur65IlSyQfSa/HsmUAsHAhLCwAYOtW5OfDzQ3h4ZKPTq0DY5QUYG9vX1BQ8P333+v1emlHio/HhQvo2hVvvQUA9fVYtQoAIiNhwh8/iYO/JFKAl5dX9+7dr169mp6eLuEwDQ2NZ0UXL27cmHnLFhQUoH9/LrYnETFGSQEqlWrixIkA4uLiJBwmLg6XLqF7d0REAIBG03j/UlQUp6IkIv6YSBmTJ08GEB8fr9PpJBmgoaHxrOjSpY1T0U2bUFyMQYMwbpwkI1JrxRglZQwaNKhv374VFRX/+Mc/JBkgNhb/93/o2RPTpwNAXR3+8hcAiIyESiXJiNRaMUZJMdId1+t0ujJDt4sXw9QUADZuxLVr8PBAUJDow1ErxxglxUyZMgXArl27NBqNuD3v2LHD6ccfvwwMxLRpAFBXhy+/BICoKE5FSXSMUVJM7969X375ZRsbh59+KhSxW61W+/nnnwPoNGUK1GoANf/7vygpwauv4vXXRRyIyIAxSkp6883UgoLLMTEvidhnTExMXl7eSy+9NGnSJAA1NTU9ly1b/uqrtYYrTkRiY4ySksaP76BSISUFNTXidKjValesWAEgKipKrVYDWL9+fWlZ2T61uo2vrzhjED2IMUpK6toVXl749VfRHm+8bdu2/Pz8fv36hYeHA6ipqVmzZg2AKMM6fCIJMEZJYYb9P0W5XK/VaqOjowFERkaamJgAWLduXXl5+dChQ0cbNmwmkgC3bSaFlZaiSxeYmqK0FDY2RnW1YcOG2bNnu7m5XbhwwcTE5M6dO66urhUVFUePHh05cqRI9RI1xdkoKczJCSNGQKPBDz8Y1Y9Go1m5ciWAqKgow1R07dq1FRUV3t7ezFCSFGOUlCfKcf3mzZuLi4vd3d3HjRsH4NatW1999RWAZbxATxJjjJLywsJgbo4jR1Be3sIe6urqVq1aBaBv374zZswoLS1du3ZtZWXlqFGjhg8fLmatRA/hA5ZJeba2mD4d1tZowS4ltbW1J06c+PLLL69duwYgPj7ezMysuLj4/PnzAOTYGZpaPV5iIuUVFyMjA6++im7dGlv0eiQlYcgQdO786I/k5t7Yty923759x44du3svaYcOHXx9fePi4tRqtU6n8/PzO3jwoCzfgFo1zkZJeadOYeJEvPoqMjIaNwLV6TBxIpKTH4jRhgZkZGDvXhw+jPLy6qtX/wTAxMTE09Nz9OjRgYGBQ4cOValURUVFJ06cALB48WJlvg+1MoxReiqYmCA/H1u34o9/bPpSaSn278f+/UhNRXV1Y6Ot7Yt//ONHr73mMXbs2A4dOtz//oiIiBMnTri4uHh7e8tSO7V2PKgn5SUlYfJkrFmDyEhcugQHB9TXw8ICycnYuROJibj7I3V3R0AAAgIwZEjjBngPu3nzprOzs06nKy4udnJyku1bUKvFK/X0tJg1C05O+OSTBxo7d4alJUaPxldfoaAAFy4gOhrDhj02QwHY2tqOHTu2oaHh+++/l7pmIjBG6elhaoqNG7FjB44evde4aBFu3EBqKv70p3sXoH7XtGnTAMTGxkpQJlFTjFF6igwbhilT8MEH0GobW+zt0aZNs/sJDAxs37795culP//8i7gVEj2MMUpPl9WrUVKCTZuM6sTS0vL998/X1xfFxnb4/XcTGYcxSk8XJydERcH4Gzh9fbvV1SE2FryGSlJjjNJTZ/ZsdO9ubCcjR6JzZ+Tm4l//EqEkot/AGCXlOThg1Kh7/1WrsXEjRo+Gg0PL+zQxadzxhNeZSGpcN0rK++tfERYGZ2eRuz17Fp6ecHDAtWu/tUCKyEicjZLCfvoJH34IT0/U14vcs4cH3NxQXo4jR8TpsLS0NDU1VZy+6DnCGCWFffopAHzwAczNxe988mTA6OP6/Pz8devW+fr6uri4hIaG1tXViVIbPTd4qENKSk5GRgYcHDBnjiT9T52K5cuhUjX7g4IgnD59Ojk5OSUl5eLFi4bGNm3ajBo1qrKyslOnTiIXSs8yxigppqEBhj2Yli7FCy9IMoSTE3JyHtgmSqvFnTuwsXl0tmq1+Okn/PTTupiYLwwbmAKws7MLDAwcN27cmDFjrKysJCmUnmWMUVLM9u3IzsaLL+Ldd6Ua4r//G5s347vvMH16Y8uBAwgOxq+/PnBz1K+/4sgRJCRgzx5UVWH48IZr16517drV398/MDDQ39/fzMxMqhLp2ccYJWXU1+PzzwFg2TJJzoreZWmJuXPx+uuwtW36Unk5du/GDz/gyBHcPeE5YADGjJm+Zs0IDw8PCcui5whjlJSxfj3y8+Hu3ngVSDojRqCkBPPnP3CDaVUV/PyQng69HgDUarz2GkJCMH48XF0BOABGrFmlVoYxSgq4cwerVgFAdHTjdvfSUavx9dcYPhwzZmDIkMZGGxuUlMDcHN7eCAzExInir1ql1oMxSgr44guUl8PbGwEBcgzn7Y2wMMyahTNn7jUmJ8PVFW3bylEAPd+4bpTkVlGBtWsBIDpavkHXrEF+Pv7613stAwYwQ0kcjFGS24YNuzt2rA8KwtCh8g3auTMiIxEVhYoK+QalVoL31JOsCgoK+vTpIwjqc+fy+vVzlHq4997DtWvYuxcAdDp4eKCuDjk5TRc8ERmDs1GS1ZIlSzQazZQp4TJkaBOmptiwAVeuyDwsPf8YoySfrKys2NhYc3PzJUuWKFKAtzfeeEORkel5xhgl+Xz66ad6vX7WrFmurq4yDKfX4+23G4/o74qJgSDwiJ7ExBglmZw6dWrv3r1t27ZdsGCBPCMmJGDwYLzzjjyjUevFGCWZzJ8/XxCEuXPnOjrKcVa0oQGRkQAweLAMo1Grxiv1JIfY2Njp06d36NAhNze3Xbt2Moy4dSvefRe9eiE7G9xXhCTFu5hIQkVFRcnJyQkJCSdOnACwcOFCeTK0vh7LlwPAZ58xQ0lynI2S+C5fvrxr166kpKQz99196eDgUFRUZGFhIUMB69bho4/Qvz/On5f8nn0izkZJPJmZuw8dWvi3v2VlZRka2rVr5+/vf+zYsbKyslWrVsmToTU1WLkSAJYvZ4aSHPgrI6NlZyMyEv36YcCAl1JSsrKybG1t33jjjfj4+JKSEj8/v8rKyo4dO74h14rNdetQVoZXXkFQkDwDUmvH2Si1iF6PkyeRlIRdu1BQ0Njo6NjN0/PQ4sUjR440/c8TjauqqrRabX19vVqtlqGuqiqsXg0A0dEteQQTUQswRuk3VVdj505cuACdDn36YPJkdOqE/Hx4e6OkpPE9Xbpg/HhMmABv7zZqte99n75165adnZ2ZmdmtW7cuXbrUt29fqev94gtUVmLYMPj4SD0UUSNeYqLHy8zEmDGwtMSoUTA3R1oacnMRHw9/f3TpAnNzhIQgPBxDhjQ5B3nz5s09e/YkJCSkpqZqNBpD47JlyxYuXChpvRUV6NEDt28jPR1eXpIORXQPY5Qeo6EBbm5wcsLBgzBcGhIEzJyJhATk5KC+/oHnbRqUleGHH+oPHGi/f39dfT0AtVo9fPjw3r17b9y4cdCgQWfPnpW05KVLr0RF9QwORkqKpOMQPYAxSo9x+DB8fZvO66qq4OyMVavw4Yf3GouLsX8/9uzBgQPQ6QB88PLL/7a2Dg8PnzRpkpOTk0ajcXBwqK6uvnLlSo8ePSSqt6io6KWXXurff9i33+5zd+diUZIPz43SY5w5A1NTvPLKA402NnB1bXwWR34+EhOxaxdOnYLhj7GlJQICMGHC/wsKMrnvOZwWFhavv/763//+9+Tk5E8++USieqOiojQaTe/eHZmhJDMueKLHqK6GnR1MH/pD6+iIW7cAYPly/PnPOHkSlpYIDERMDMrKkJKCiAiTh55lHBoaCiApKUmiYnNycmJiYtRqtVJb8FFrxtkoPUb79rhxAzpd0yQtK0PXrgAwdSrq6jBhAvz9f3fjuYCAAGtr61OnTl29etXFxUX0YpcsWaLT6WbOnNm7d2/ROyf6bZyN0mN4eKChAdnZDzSWlyM3F56eAODjgx07MH78k2zeaWVl5efnJwhCigRXfzIzM+Pj4y0tLRctWiR650S/izFKjzFyJHr1wqefGq4aNVq6FBYWmDKlBf1NmDAB0hzX390NWop5LtHv4pV6erwzZ+DvDycnjB0Lc3McO4bTpxEXh5CQFnR2+/ZtBwcHrVZbUlLi4OAgVo3//Oc/Bw8ebG1tfeXKFXl2MiVqgrNRejxPT1y6hDfewLVryMmBjw8uXmxZhgJ44YUXfHx8GhoaxD2uX7BggSAIH330ETOUlMLZKMln27Zt77zzzpgxYw4cOCBKh4cPH/b19bWxscnLy7N9aHkAkTw4GyX5hISEmJqaHj16tLKyUpQOFy9eDGDevHnMUFIQY5TkY29vP3z4cK1Wu7fJ4zqb7+zZsxERESdPnnRycpozZ44o5RG1DGOUZGXk9frs7OzIyMjevXt7enp+99139vb2wcHB1tbWotZI1Dxcfk+yCg0NnTNnzsGDB6urq5/wuUyCIJw8eTIxMTEpKamwsNDQ2KlTJ1dX17S0tDZ85DwpjbNRkpWjo6OXl5dGo/nxxx9/+516vT4tLW3+/Pm9evUaMmTImjVrCgsLu3Tp8uGHH6amphYWFr7//vsAysvLZSmc6LE4GyW5hYaGpqWlJSUlTZo06eFX9Xp9enp6QkJCYmJiyX92hnZxcRk/fnx4ePjQoUNV/9nU3rDCqaysTLbKiR6JC55IblevXu3WrVubNm0qKiqsrKwMjQ0NDRkZGQkJCQkJCdevXzc0duvWLSQkpEl63pWZmTlgwAA3N7e7T9AjUgRnoyQ3FxcXT0/P06dPHzp0KCgoyJCe8fHxpaWlhjd07949ODj4kelZUVHRsWNHw78Nt0JxNkqK42yUFBAdHb1gwYKePXtWVlbeXUPap0+fsLCwsLCwgQMHNnl/UVFRcnJyQkLCyZMni4qKOnXqBKChocHCwkKv19fX15s+vKEfkVz44yMFDBs2zNnZOTc3VxCEfv36BQUFBQYGent7N3lbXl5eYmJiYmLi6dOnDX/vra2tMzMzDTGqVqvt7e3Ly8t/+eUXJycnBb4GEQDGKCkiIyPj+vXrHh4esbGxffr0afJqQUFBSkpKQkJCenq6IT3btGkzatSo8PDw0NDQtm3b3n2no6NjeXl5WVkZY5QUxBglBWzfvh3AkiVL7s/QvLw8w/NET5w4YWixsrLy8fEJDw+fMGHCI9fYOzo6ZmZm8vQoKYsxSnI7d+7chQsX7O3tx44da2hJSkpaunRp9n+2iLa1tQ0ODg4LC/P19bUwPJT0MdzdV5SVRVZW9pe8aKLHY4yS3GJiYgBMmzbN3Nzc0KJSqbKzs21tbQMDA8PDw8eMGXP3pd/zSmYmrl2TrFaiJ8AYJVnpdLq4uDgAM2bMuNvo7+9/8OBBHx+f5l5wN2wxymN6UhZjlGS1b9++srIyNzc3Dw+Pu42GJzW1oDfDJvq8HZSUxXvqSVaGI/o333xTlN44G6WnAWOU5FNZWbl//35TU9Np06aJ0iFjlJ4GjFGSz86dOzUajZ+fn7OzsygdGg7qGaOkLMYoycdwRH//xSUjOThApUJ5OXhLMymI99STTC5evOjm5ta+ffvr16+LuNeyrS2qqnDjBuzsxOqSqHl4pZ5ksnNnr6FD84cPzxB3v3pHR1RVoayMMUqK4WyU5KDXo1s3FBcjPR1eXmL2fOQITE3xhz+AD2QipXA2SnJITUVxMXr1wuDB4nRYV4eICLi5YenSe42ffYbOnfHuu+IMQfSEeImJ5BATAwAzZuChPexbSKdDQgIiI3H48L3GY8dw9qw4/RM9OcYoSa66GikpUKkwdarIPXt5YfZsaDQid0vULIxRktz33+PXX+HjgxdfFLnn//kfVFcjOlrkbomahTFKkrt7RC+6du2wciWio5GTI37nRE+IMUrSys9HejqsrTF+vCT9z5gBT0/8139J0jnRk2CMkrS+/RaCgPBw3Pfsj5a4devR7SoVvvkGx45h1y6j+idqMcYoSUgQEBsLGH1En52NXr2wadOjX+3fH7Nn489/Rn29UaMQtQxjlCR07Bjy8tCtG4YNa3knBQXw80NFBQ4deuy98599htpaZGS0fBSiFmOMkoRSUgAgIgImLf2h/fIL/P1RUoIRIxAb+9hlp+3aYfVq6PUtHIXIGLyLiSS0ejUCA9G7dws/fvs2/P1x+TIGDEByMiwtAWDvXiQmYsMGbNqE+5/NPHkytFrx11QR/S7eU09iCgqCRoPduxsjD8Bf/oKcHGzZ0uyu6usRFIRDh9CjB9LSYHgQfVoa/PxQW4u//U2SFVRELcCDehJTZiZSU7Fy5b2Wq1fx88/N7kevx/TpOHQInTohNbUxQ7OyEByM2lrMnMkMpacIY5REFhCA6GhcumRUJ0uX5iQkwMYGP/7YeJyenw8/P9y8iZAQfP21KJUSiYMxSiIbMwajRmHWrJbvSL9o0aLly/uMHv3PPXswYAAAVFRg7Fhcv46RIxEXh2Y+hplIWoxREt+aNcjIaFwx2lwbNmxYvny5iYlq1qxib28AqK5uvNA0cCB27bp31pXoKcEYJfH16YOPP8bcuaiqat4Hd+7cOWfOHJVKtXnz5tDQUAD19QgLw9mz6NkTBw/CxkaSgomMwRglSSxeDCsrREU14yOHDx9+66239Hr9F1988fbbbwNoaMC0aUhNbbzQZHicMtHThjFK4qisfOC/VlZYswbr19+7TP/ba+NPnTo1bty4+vr6+fPnz507F4AgCEuW7P/hB9ja4uBBdO8uTd1ERmOMkrFqazF/Pvr2RWnpA+3jx8PPD4cOAUB5Ofr1Q1zco3vIzs4OCAioqamJiIhYsWKFoXHhwoUrVrw+ePCqvXvRv7+k34DIKIxRMkpaGtzdsWoVqqqQnt701a+/hpUVAGzYgMuXMWUKZs5ETU3TtyUlJVVWVo4bN27btm0qlQrA+vXrV65caWZmtmCB+5AhMnwPIiMIRC1SWyvMmyeo1QIg9O8vnD0rCIJw/LhQVPTA286dE/71L0GvFzZtEqysBEB48UXhxImmvW3fvr22ttbw7x07dpiYmKhUqm+//Vb670FkLMYotcT588LAgQIgmJoK8+YJdXVP9KnMTGHAAAEQzMyE1asrdTrdw+/Zu3evmZkZgNWrV4tcNJE0GKPUPFqtEB0tmJsLgNCjh3D8ePM+XlcnzJsnmJo2DBrk4+XllZube/+rJ0+etLa2BrBw4UIxiyaSEmOUmiErS/D0FABBpRJmzhTu3GlhP0eP5jo7OwOwsbGJi4szNGZmZtrZ2QGYMWOGXq8XrWgiiTFG6YnodLqVK1cOGHAHEFxdhWPHjO2woqIiJCTEcII+PDw8KyvLxcUFQHBwsFarFaNkIplwozz6fT///PObb76ZkZHRq9d4X9+kVatURj5Y6a7Nmzd//PHHNTU1arW6oaFh5MiR+/fvt+T9nvRMYYzSbxEEYcuWLYakc3Jy2rJlS2BgoLhD5OXlTZ06NTs7297e/t///rcN7/ekZw1jlB6rsLDw7bffPnr0KIDw8PBvvvnGcO5SdHfu3CksLOzZs6eFhYUU/RNJijFKj7Z9+/YPPvjg9u3bDg4O33zzzXiJHjNP9Ozjxo3UVGlp6Xvvvbd7924AYWFhGzdu7NChg9JFET29GKP0gOPHj4eEhNy8edPOzm79+vVTpkxRuiKipx0P6ukBVVVV7u7ubm5uW7du7dy5s9LlED0DGKPU1NWrVw1LOInoSTBGiYiMwo3yiIiMwhglIjIKY5SIyCiMUSIiozBGiYiM8v8BBq3eN4zITYIAAB+5elRYdHJka2l0UEtMIHJka2l0IDIwMjMuMDkuNQAAeJyFmWdUFGvw5hlyjkOOZgREFCPIW4MoCJjjvYqKCiYUEBVFFBEMiGSQHBQQJaigZPCtVxGzYsSMCYmSRJDMztzzP7t79sv2lzrT3TPn13Weqnqqp4NerRPiHwpC/+eQ/5/P/hxxAzN+FBGVcDYURBFx5zmCyPm/TvwXRf/fKGnwXxT+3/f9zw/9/67LCHGEOMLOwiI7hUVEnUXFdgqLiTuLSxgKi0s6S0oJSUo7S8sYCkvLOsvKCYlxnGVFnbnC/C+Kcfi3i4mLS0rLioqpC40VEprOP8v572EqmySZt5gjE5eNxu7cCFSbqM+sMq/TqrmdNNncit17bwFlS6tI/EMOy65OgHsqkWQodg++9vlIso4a4TzrVJz0XRx32R/Be9NC6IIjlTgjvxVL++Rw6agqc/5WQFtW7AL9s0VommOGwg+7wfO1Fkv6xYXYfXFgKt5Ge9ZGQx9T4dG4elqquRM66Fly8XQnCV33B0otVaFxPwWOAFd+i7Q4c4txYGs3x+LCB8GYfVSPBZ6upEYSb6jNG8Lm8aZCpWgduTlfmLWqnwdVjzRib7ML6abXZFzFePzYkYxfT0ng9rSjGJZ7hhYFlmHSuHb88VkGTe247PHSh3SM1j7QXVqIU5Ks0fbpIPDKNVimixScKE6AwaVttIZ3DuouqPFy332ng6d3gs/EOPLStpU4Lu+BLbdU4fnmW8ARpF1+mbEsE+c5sPtTQ7BrWzyGR+iw0vQ0um+RLM7bMZfJ/OsAn7YnE3eFQVRYlQbBq3eRpq3bcNGbb6T6xgwMkU7EzdliGPLXF7VXn6VbHSmO6rbgugdyuKlRjZ14e4UWrN8OojklaHFZH/9Z0QWPWnWYlz8XDh2Igy7tXhrcFgfLXijzPOoaaOcVZ7juHkMa6nvImTNt8MNSB17uR+CICGjdzkiznyqOTHd5GLpujMAqKX322L2Iyi9/R2uDCRtSMgX7Yz9JrbUwm+IYDltDUsnhwx5YlInkis8UjDdJwXJpGdzQGYDNR4/Q/LwyHLe4Bz+86KFXB5TZ0vR7tC3PEy4q30SPibY4yXwAngSps9L3XHinFwfcyb/pho8RQC1VeVVlw1RawwvE4v3Io2cfyKPZw7A0RRFKn+YDR1RAy9snxQ46OrJn9yPxjVE4brumzwyiblKDAx/oP6GEVd2cDtu4n8mvRmE2/n00HA1JIrZG7li/6BHp6TPCMywVffwkMarhGHJ7j1N1tTL8496B3ieGqVugCpsleocG3/KCA92FWCvDw9SXQ/AwRZOlFMrDqyPxsIs7QG1MIuCCDZfn6dlOrb7tgZoZwcTN6gtZrtcPb7xVIF6iGDhiAtqGfAm2L3oJWz4xFC+YRuI2Zz02pfwWLYUmml41j8m/mwL705+SkyuFWNulKNgwJphoTtqFby/fI1LJZqhUmYQqnnK42CgArdqO05SIChxd3YlJWT3UYJ0qizz9kF7bsA9y8SbKN89Fn4kjkOGqzvTVZcDkWgK483X71SwUHI9yeQ4ddTR6yAVKesNI2clGYq79B2rWqMLyOeXAERfQ5ipJsMC/jmzt4SjcdyEM33fqshI1SmvVvtDoS/PYgY9mcP3SC+LuJMTutMSARFo0eTvNBbPya8isvMn451Qiv8qk8N7nE+h14xR1DalEZc9O3BYui+nSqsxg3xPq/sUdhrwLcV434NC8fhh212Dt+yUB3yRB2uku+qAsBDb1qfI2wUc62Xo7zIu9SNobm8lz79/Qv4kL5q78KpMQ0L5PlWYi2o5MaXEQpqnGYUqtFutYkEt/Zsqgd94cFtfCA+X8dLJ12wAmc5KhpmA/oWZbMKLuLSEHp2OCUhyWbpTGgi5fND7jT2e9RFya2Yp9raN0iYcme3Api+oYekDRrzI0Fx+HFq//wGmeHrNPUoOvGfFgcG6AeifGAK9FhdeYy0H/RS6Q2htHKnkdpDmsEyTvK4P+otvAkRTQnn0kwZJtF7Om3gjUsIvETI4um72vkrLk39Sj3oKtMpoDH7sryePsYSRj4mHZwSByWMMVF4W+IpvfmeAd2USMyJHDSfE+WKl4hFqYIu5QbkaXN3+o2HMNlm9yjRbU7wPyvRgTOWa46nQvaBjrsKIyJVC34Xewyx20iK+wS6+4vNrFX6nwMlcIYOdJf3UXOQhdMKlIA1Ja+T1BSkC7+rEE65y8hAU9jMR9Y6JwMFOPSYsW01H3EdoSPY9ViVrCzjf5xGm6EOMqJ4HkBn8SxXbissbPpG7nFFTUTsF+fUmUVT+E3pfP0KrVlbg/+ydOduunZrfUWUJECfV/7gH5q0rRfuEU1PPqgz/rdJh3nywM98eDXckv+vpxGHzo5fLc279R14mbYW1vLLH6t5e8kGwDekUTxu25AxxpAa3dJSl27LADe/MsHC8URqKSiAEbO3yBLs/opHLtVmyZsjVEWt0lY7kc5jQ3EQauBJHWDXtwKP8tseyYihvOpaGPiBgGLfTFupbj9JlvOR56+gsfL5bBehkuuxxfTEUeeYJvTBFum2yK3S4DoF+pyaxdFaD2zXn4PaGDXpoTDhdGVXhPhXpoyQ5XmBR+hrzObieveN3Qt1UbPi/kV5mMgNZhujQT+2jPfm+IRPFXUdiwQp/Vz0qlwcnD1NDBit3InA9vE8rI6tdC7GRJCizuDCBv7u3Ggt315K+IGdq6pOK7j0K4qNQHZ5w+QuumlqECtOK41bIodEuF5QwV0gp7L0icW4hNUWb4TrMPHpprsYDdymC6NwGk2r/StKxQUFVU5fmf7aR9G7bA/JUhROxDP9k+0gFm9voQ8JpfZbICWj8xeda+yoE9uxWMhSHxWLBXj62Oj6XRfrI4vsGSda9whHsHr5AJR4VYy9NUsFc8Qy7dc0Mvmy/kR64ZdtqmoVIRB3WKvDAlI4RG11SgdG4TTlqjgLErVBl3wXkK/Dr3V+LnvJCL9wN+g7KMHlN6MA4uOsTCqZMc/Ho3GZ6Kq/BORcrgh/5dcKg7kJTe/EpiZvXA08/q4EdKgCMnoLVOl2LVDxxY8/ho3MWLxgkf9Nimvks03kQYbUbnsfNpNjBG9CrR8hNiXzJTodTQj0jp7MJm42bSxZmKdS4pGBoujBMCvdHzqw9doF+BcVua8OU6WWxMUmUr716l22S84G1/IW4MMsRhTh9MeKnF4iqUQeddAthBHfUVDQOTAVVeukU7zb63Ca5tjCCir0ZJ5PtfYDYyBu4jA47ATMoHKMqxnB327HJUEOq4xuH8NdrMjl2lpWVieG+qJfu6eCak9VWQ9b1CbIliLOT0RJPmjF34NOcduSM/HX/+jMNjSRJYoeiP3Lbt1Ph5GfpaduGqv+10zCcVtjvwKp0RfgDqFhXitVtT8YDuABhc12QmQTqgkhEH43Z00KeesSD9UIVX6CKE/cke8PboaXJpoJFUlA7Am89KYDvAn2UC2yv/slyMtRU7sMjWc6gYw59npTps4rNL9M+LZlqtYsneH+VBcmA1mTRmFDedToa2zGAypdgVZ0V9ILeyTZBXk4AZ02RQ84AfXis4TM83MJSvakV68je1fKTJ/lEqpbVaHvBTqBRx7RSUqOyDVx7azPaILBSoJkOjqRD2CUVDrJkqz/hYHdUo3whOJ9NISdtHsia0FfJmKcBf2UrgKApopR5Ks0Vt9mxPVyi+t4vCLl891mJ2kWad6qTWslbMR5MHZ548ILWMw+YkJEJ1azTZyXNDCas60hdpgrOXpGDHBA4WXfbBS17+tMe6BO89akXxP0IYk6/McmNLqFs/P7fHizHz+0y8d78fjo1qsev7VCBvcRz4v+qkpD0SjhxQ4ZVP7aZSzrtgs3o06XH6SebV9ILFKVWYcIDfE5QEtOvkpNgR/UUsYn4I/usRiQsjdJlkYhTdPomD2s8s2cSZS+C3bwEJDh/B9kOpIHIhlLgscMG9478SNQ1jPCKUhGOzxFB/hQ/mfYqkd2wYXoptQeF9CjgcrcG4m7NpWLQr5P3hu3R9Q3Qtbofx4QZM/IE67N8cC2lSorjSPxY+NSjxFlwVQbXDO2Dbm1hSLfadZN3shCAnGRivw3fjygLafDsJluqyiC0vC0FtzzAM2KjLUt5n0COjzfT6TSt2vG8e8CbVkGxtEZa/PQmkTYNIhOl+/CD3hrxZaIq3ryein540lo31xTzuFtpoLqiyXyi7v56qNqoyOc8i+s3zIPg0FKLfR3M0XjIIUwo02eO9qiDtmgSTT7TTWIiCK99UeUL32+nTuo0AswOJYVQt2bmvA5btkYLp9XzdqghojXZLsbO/7JnJxSh8WxmFlY902YXym7Suc4B+JPPYi1dzQCWxkkw/JMQ+xCbAkbyzZHH2LvSp+kgiXkzBTauT8B8PSTT4fgiHQ/3oVF45jkS04KsOCdTu5bIF16/Tz2aeEL2gEBVGpuITkb+gsEOLvfNWBoXhOEh+0UQfKkXB8imqvPs2DXSPnCvUTgolxhe7iUV9F0R368PzKxXA4Qpoi86KsUYXe5ZeHYUqxqE47KDHsp3y6NqLzdT9rRWbEW8FbBEjtjs4LCQ4Ef41DSa7ZXejx+PPZLTFGJ2y+ZvOUSmsNjiOcdxQ+tiO4rm3rfg6qY+mH9FkoebV9Jr6TsikxWh20xyLHf/C141arK1CAkyWxMGcZZ3U6u5ZMBrL5V23/kV7zTeDU0EUKTjWSSx626BqnzqseMH3CaoCWkVPcTbnlQPbujoSsy6FY7WxAdMxiafrrg5TOS0rduusAwS+ukqmyXCY/6QMgKoT5NpJN7zY1k3EYo0wcHManpwihI/93fHz12A69wffa7V/x3tHuGhwTZVpt+fRq4EeUDKlDBP+mYQe23sgpkeXrWuRhiGjONhZ0Unp0Qj4oMTlBel9pCu2OcHhujgSdHaI7KtuBr8n+jA1oAo4agJaMlucBXOWsJUbonE3hGLvE32mr3eL6t35Tq9tJmz47hToKn9OAseLsLis88CVSyZC9W743qaWjD42ROn8VJT3l8FN7YfQJ9afqk6oQPn2VvwyKItzDFTZDPFbtKHCC8qHirAzdyaeXjQIbzma7Gu9PGTlJsCC8e3UelwoWIuo8eLutNDqf13hpVw4+XriJ/kypgvGeqlCtkMZcNQFtCfaJZmxxGL2+FoE6gdF4UCiLnPJqqRb3YfoqgPzmJryNAj4eouo8Cfv3SmxMOFmCGm5uBsdsmpJ0TpT9DBMwnu2shgj5IvprYfo2oxKDGttQc0NbTTPXp1ZOuTRHEsv6Nl7E88dMkc/flcq36rF9jI1kJFJhIcJP+kabjRInFLj7elrpAfnbwf3smCS3N5BNvb8BnMvTXj8ia9bDQGt41xJpnbanl15GoLXgiKx8aYum7kxhx5U76V7cyzZ0r/8KlOqInciR1Gbv5tuGRtGPozbjvuNXhH9TWbo/plP+0sac1b54iP0oRb1Faj2pRUHjw5RmxJVtsGmlOZOPwCLykvxbvE0tEwfAKNv2oyzTQEuqcSD9c4hKtYfA+XzVXhOP75S9VMuED0hnGwO+UYqnnZBepwCvBbm91tNAW0mV5rpJjuwFf+EIe8Lv4ON6DMp5VyakfeS2gUAez84DVZnNJFyb1EWkRQFR6uyiMVDL1SNfU4szhijcXYqyqwUw2P/HsP2XbuoBN9rtRt2o4UxBwelFdkTmfu0YtAHHPEGBoYuwlnjR+CFgzpbeUERwufFQ9aTLrrzYDicu6nKu6jYS7O63WFWTiDRHXxP0rcPwvtkOdB6UwAcLQFtXoMky9xqx2T8wvGARjhK1uizhVtD6Np/e2jSYmBtS+1AZsEdUtUuwrpZGuyvOUeSph/GiWe6iGa8ES7NSUX1/X00zPwomgRF0UK7EvQZ+YU3eiRx7hgu+3fFHVr2yQ2SJUvwy5tpeJDXB64OWuzuB1noE4+F2Om/ac63MDAnKrzLtl20b+ZG+F5yiATsbSIDX9og4IIazNbjezBtAa1WhjjTXOPIHsuGYZxQBI5O0WV32wupwo8e+uzDPBa+dxYcv3aLDNZw2PY5CVBjFUpS/XbjROvP5JivIdYJJeCd3RIofuogGnLdqdvsckzMbMSrOt10+nlV5tiUR/0O+kBQXSFeeT4TRwb7QMlfm5mVKsDKdYkQHtBOr2+NhFV5XJ6N0Ud6d9lmGG8aSpa/byQuju2wwFIRPOv5PkFHQCuySZxFfF7E/haFoZB+GKaf1WOrHqfQlJSfVKHFkv20WgiLKx4QczaCnsPJUOscQRba7sAdn7+TGL3JaNWchMtyRHDenMMYs+sE1WYV+FK5FfMTVXCgicuuzqik3zZ5QMZQKTp/mobHuH0guVyHkYVSECwbB6NxQ3TGsRAwXMbfImN76cf8baDVmUzcA1tIm3Q7TL+kBDK+fDeuK6CtWS7BTE45sqKgCJSSj0CvWzpM17KSwoIeOu6FJZs+aToEhlKS0sHf0PX4/XFONDnz2hVzfN+R03ON8KpfAsYoyeLpvoMoX7CXrllTibNfNOOy+8L43U2Nif24ShOOHwDVV0V4rs4URV37wbxEiynVKUFHcgJw7Trp2mnRMNVZlRdx+gvdtmwnLEmJI/eKO8je6t9Qm64BEbV8Wj0BbdywJIuycGQ/jobiIYNIDJfSZzUkk94o7KKFawmbeQagLrKaiL8UZqFFSfDSN4I8affAsAs/yNBjE5TvTMFrs0TwmcUBrLE4QKMjirH8Nt/VLONg3msVJh98gxoGHwS7zkLc02iOPVcGwLNKiykbK8GLcwng96WLzjGKAINGLi9X5jeVi90C6U5BZFtQPcl+1QFzLPkb+qZS4OgLaNfHS7Azt+3Y8W+hKP8oDEuu67KpLecp16efmmZYsakvbEFzVyl5tInDku1SIOthKOl8y5+8pQ3kSdlktJiehNGZwmgtdhjD3P3oinOVeHF8C/LmyqGmrSrb4VFCpTd4Qf6OCjxTbIrT9/bBi1+67MwsWYiNiYUCDQ5OroiAEUMVnp1WHy085QwzWiJI3LgfRNSgHV7+kgO17XzdGghof2yWYdv67dmE0mDksRj8wN8djhYVUzcpIex2m8f+0TKHaeZ3iPNlYba25DxsbIkjJ/k7b8fpWhK9ywTDDsZjcbs40nfHkWO3ne4NqsA2hS6cW9tLJ7tzmdadcqo0sg8CtUvwm+xsfBrWD1sfazGNu8owLT4elC2EMW5NLDxtVea1TvtNyY7dcDE+mPS1viWbu3rBUUMaGlYWAWeMgDbZUJwZqy5isy+GY9yrUDzD1WWetVdoeuUvusXQit0VmgdXd1aTJ0IiTGxFMnTeiST6np5493EzcYswQgOvJJwYKoY5LUfx8usT1ONcBf4Q/4UJkb+po6E6U75QSh8J7QejpkLckjAXvaf+hdn1fCU0yECSUyzov2qhlp9DYfIyZZ7cvM/03oktYHwziJhYdpLZ67ug6qEqtK7k+9uxAtrRZHH2R9We/W05i/6PgrH9jD5rfJpMxbhPaL+DNau8Mhfc5FpJ6koxZrLhPLTZJpMXH3wwvOEFmWVngpJKKSg9SwJzegLwQak7nb2sFFPru7HiUjfVvK3EfIc+ULkAbyg3KcY/rTbY0M/hzeSqMxc9MWhbEQtGBSN0x6wweOOvwltU2UD3znCBC6u9ya+CGlJd+AeO7ReBS3vzgTNOQLvophAL0F/C3v0bi0knTuJotgEba15ML7kW0+NiPAZ6c6B+cQMJUhdm1w7FwxPJUDKJ64EHf74g9+8YoW1xKgqJK2PX2iPIDh+g5zMq8GN4M05X7qYax9TZEdF3dPmhozBNtATDr9ih05AQL3m7Jtsq1ktGpiRDYFUrdfA7CeOmqfECR5BeCVsNB9zPE3HvH8S2uwHMqAysvc7P7fj/lJAgwXpXOrKCxBgsuh2G51z12dNxN+kdj2Zact6KYcMcWHHoHlnYy2EJ0xJgxbR4ss90N+4arCM7qg2xxCsVE3sl8N72AxiqeIRWD1fgzhVNeLREFs9xVZlaRj4tWLMfEtRKcHKEKfqE98PyHi1mVMuFt88SwffzL9p5ORzqX6rxhsJ66OB0V9g/J4o0FTUT2wudUDaiCd9E+a5mgoB2728x9mSnA7u5OAJD/4RitJsu63+dTQvke+mRsfOYUZYltEgUERfhUazjz7Kc96dJzI4dOND1mVi9NEL1XYn4aJEUztfyxW0vj1NOOmKJZTMWnpBF3w41dtK2lM6c4QHdE0rxVpExLlk0AE4vtdnMMmlwsUyEA1F99KRTGNhRLi/gdgNNX+QEBjppZJPHD7I7vAVgphz8tGbAmSig1ZCQYk+zHFnT+CgsLotAhWB9Vse5QnO0/9JHFoRNMydgklRBzl4QZrtZEqxSjCBRph6oENhEOt1NMGtrGu4IE0V7My88tiqQrugrQaf8JrytLoPi2Vx24lwB7Z64H6KlS/DHtcko6T4IShVazHIFF6K3JIL37Vb65Xs4SE5S41273EM1wBkefT3L93ktxOhZO8yMVwcHN35uJwloTTdIsWNL7Bg5H4paB0KxeboukzPLpS1vX1BOEWGVhjNhslYLmbxdhNE1UdBqWEAq1N1Rr+85Cdk8Eb8+SMTy02L4asYxjGjcRQuSizHvzG+0LPxDt0UpsZljKqjbdm/ImHsT4zsX4AnjfqifoMFeHVKF9K3nIby+h66vjoVdUsq8kYM/KHuyH7KKTpD1Im/Jl0sjkC8pCUWRN4Aj+C9O3nCTKFtjY8+aLcPx+cMgXBKvx9aPyaQ9s1/Tqb+B6URZguKr1+TNemE2kHMehK5EklVSHhikV0MmpRlh+e9EpBflcMc/x5B17Kde4ypxWLcN8/w5+FhTlRWZ3qXfPh0AuRnFuMbLGp84DUDKTi12u1ACNFISIDB6kLYah8I2UOXVTGigLpGbwNo+mkx2eUlGSAfsPSoJtTf4s2zyf+9vR8WZ8gpHlqgUjAuaQ3FLoR4LCk6k8PwX3fSYMN5ja2gPriLxY4TZj/cJ8DY0lCyLdEcni88kIWEqFo2moN9zCfx76TAaWXrTc9llmExb8OONAfr0C5cFnLhDXZIPgc+5coyVmY331o/AQIMOezNTBlyfJcDc6UKY5R4JX3VVeZ0nW+nh2I2wjD8vJ/a/JdlC7TDwRxpMi/m0RgLah/nSDOpt2ca60/hrQSwqntJkAZ2htNJFC4OuW7CF3HUg6xZIVL1H8PzaTNh26AhRmr4Dr93oJYHVplgnHoft5WJ4rcQdqfFJ2tx2G8+Vf8NipwGq4qTN7lacpOVH3CDGGvHqFzmcNdoGbrfGscVi48BtJBZu5oni6+o0kMxV5sVclsDVIc7gmXeWlJ9tIrYOrRAdqwRJk/gdzFhAW6onzfbGOrDCWUGYuz4ac7brMqvbGXRSBQfVagkT9yXg9KGMPBcWZjPnx4FbVBjBR55Y1f+e0MxpGLc3CR9ISeNMk+P4Quc4XR5WiQWtbWgt2UY3jlVjUY9u0X8qPKCnvRwPXjNCoy/DUPRdm7034cK9awkgM1UUu6/EQa29Cm9+cje1cd0Kec6niGJuDTGe8Bsut0jC9bhC4JgIaOVSRVn1Z0cmNBCGe/j+9nWIPjvyMJbm2kqg4woeE165DGboJBDuCmEmrpcBM0W3koecg/jcpoPYlxijr1Iafvgjjv8ud8OHGEqfGFH0kKrDdHExzFfRYudks2jceHcIFaP48aMaGhj0wd3T+qxztizUhMVC3IcBOjEqAkx/q/DCbVrp8Jrl4KrpTz7FtBOVuq+wtk4N0ByBM+U/JdSLMfLNjv02i0bRR2fRZ54BSw3MoMfevaSuodbsuJYF5M7+Qv42iLC/D2Nh6ZxkcjnrEEYa15GR1ROx9HwaunYJ491jvnhnTQgNGCnC1TfaMP75INVBFUaePaC8ie6Q8PMGjiuzQZs5A/DHXYOdTJeE7QHnYergLzonPhg8vLi8w6otNF5tC6iOO0pcg+uIbdAfcDdXA7M0/qZjKqBdIi3MgnExe9d3Hme9P4MhfQZMpbyShsW/pLll1mz9QyN4JP2WFO0WZXuGYqDCIpYssPHCE1VfiHHFBJw5mow9RVJoJOONWa7b6dCeYhT3bsIY114qXqfCzua8pOTdcfgx+yZKytvjuGAOz7ZSnakocKCQpUBz5ifqqHgKmjrVebdL7tAS43WwzSSUvLnUSmZINUPwHhUoceY7xqkC2mlTxJmwtiOb2BaMkbfCccJzXXZPLZkaNIqgnR9hHuvtQdolnfhxOMxhdgpMdfAirRruaDrxK7npPxXH1Sfh4BIZVBjvgddkDtAXW25hbvw3VKv8Q02a1JlV6BWafNEb/prewlrLMSj/bRDe5PId/zZFsBqOh7ZjI7TGKQJknnB5gfuF0Er7X/jmHEKq/qkn8zc0gYeEPDzI4FeZ2X9vP5rE2JeIRcz9SCSWDwVj6joDljwaS8cr1NP75/lK6FoAq0/eJ0t2ijLvb0nwjRtD8p288eTcX2RcliHG8H3CmD0iePP0QWzFEzTZoBSX327EQ1W91PAxlznWUDrP4wDs+qcE88Pm4qbDA9BzTJtxn0nDad94sOR204mpYWA2wuVdLP1BtS5uAKPM48R+5w/y3OsXTDsrDxYpZcCZJqCdWSzBmpY4souGYZgzJQLdsvVYxp48CvHdVOYFYXcXzAHDpCpyaECYVU9IgFvaEURhqScqb/1OMn+b4t41KVjzVgIPS3jje3Sl73JL0P9TE77N+U5fX+Uyy2xKzY74gFFpMTomm+OwtBAvoEWT3b4iC1PuJ0FT+R9qYx0KxtJqPOWaTlr5dwMc3XSOtHE/kV1+v6CjTRFevikGtf8F1vHABLVkwC0AAAHGelRYdE1PTCByZGtpdCAyMDIzLjA5LjUAAHicfVNLjtswDN37FLpADP7Ez3KSDIpBMQ7Qpr1D970/SirNSLMZyzQk4pF6fKS3Vs+P6/c/f9vHw9dtaw2+eCOi/WYA2N5bbdr59dvb0S73l/PTc7n9Ou4/G0rDnjG5PmNf7rf3pwfbpZ1oRw6C3k68oxGBtxPuKO4gM5jaMdwMJF5B4moA7QQ7C0Nf7uHKCrtoYGBBoytZktmNwr1PpGTSjHcUhQICWVA03AmQZQH2TAk7OXOXyh0kaJEuIVddaOrjbtUe6U7CxMqq5euk7j6h9oCGmTrXjiyZWAWJZZE4od5u5XZBTlErV5duWpSzeIMla5ROpYkJcRIMRAEdSdFJbCIR/nMNgLAsOiCT0tAWg1AXKI76e9Ljnkj05CEPkSl4uR4pqSagI6T2SQ9c+6hORMWWLiEP8QNSPypxkwdXk7JvZitPycuLnQfRYEGREVmQW+R9H8DX4/ppuh7zdr4d1zlvtWjOlKTxnBspm8NRq88RyEPT2WhJs9lMSvPZMMljzKZg2Sq9lAMXgak+SIuQMjy8CIbjI4syRWLGFMdE9FWRtf46P//V3G//AL/ZwTizd7A3AAABBnpUWHRTTUlMRVMgcmRraXQgMjAyMy4wOS41AAB4nCWQza3DQAiEW3lHW1pby/AvK6cUkCJ8dwUpPrDvhviGYeD90L29PvuNe3vuB+/92d77Ta/P33c75mk5Z/qgM6eojwOnUAJXIRYX8JhnEsm0cVCxgDZLdwseVcE5yBd0SFxVhBBTM9XytDZlYaceNNM0aTnY2JcKFhFXWQWzSrcSQp61WhBmUqyWyHKa8OTKi0ks7SiWlNQo1eA15MgI7STEE7qOClu7KscsgiKJcfBJDlDHIVkZctY+lAnqM6xj+bv71R+KbKKBLEkdHJ50dYO8pOURTDpWJiRHzZDSzFH3z7B/Iia+f396aFLjBFvx3AAAAABJRU5ErkJggg==", + "text/html": [ + "\n", + "
my_propertymy_value
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Let's generate a toy dataset with a single molecule\n", + "smiles = \"Cn1cnc2c1c(=O)n(C)c(=O)n2C\"\n", + "mol = dm.to_mol(smiles)\n", + "\n", + "# We will generate 3D conformers for this molecule with some conformers\n", + "mol = dm.conformers.generate(mol, align_conformers=True)\n", + "\n", + "# Let's also set a molecular property\n", + "mol.SetProp(\"my_property\", \"my_value\")\n", + "\n", + "mol" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "d5b6aa13-3951-461d-b4fc-dfcaeb169301", + "metadata": {}, + "outputs": [], + "source": [ + "path = dm.fs.join(SAVE_DIR, \"caffeine.sdf\")\n", + "dm.to_sdf(mol, path)" + ] + }, + { + "cell_type": "markdown", + "id": "a79bf673-1bed-4c78-be92-e98a10cf5ec0", + "metadata": {}, + "source": [ + "This being a toy example, it is a very small dataset. However, for many real-world datasets SDF files can quickly get large, at which point it is no longer efficient to store everything directly in the Pandas DataFrame. This is why Polaris supports [pointer columns](./dataset_zarr.html) to store large data outside of the DataFrame in a Zarr archive. But... How to convert from SDF to Zarr? \n", + "\n", + "There are a lot of considerations here: \n", + "- You want read and write operations to be quick.\n", + "- You want to reduce the storage requirements.\n", + "- You want the conversion to be lossless.\n", + "\n", + "Chances are you've no in-depth understanding of how Zarr works, making it a big investment to convert your SDF dataset to Zarr.\n", + "\n", + "`DatasetFactory` to the rescue!" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "2955c572-6d1d-47ff-8101-5c2781fc1c4d", + "metadata": {}, + "outputs": [], + "source": [ + "from polaris.dataset import DatasetFactory\n", + "from polaris.dataset.converters import SDFConverter\n", + "\n", + "# Create a new factory object\n", + "save_dst = dm.fs.join(SAVE_DIR, \"data.zarr\")\n", + "factory = DatasetFactory(zarr_root_path=save_dst)\n", + "\n", + "# Register a converter for the SDF file format\n", + "factory.register_converter(\"sdf\", SDFConverter())\n", + "\n", + "# Process your SDF file\n", + "factory.add_from_file(path)\n", + "\n", + "# Build the dataset\n", + "dataset = factory.build()" + ] + }, + { + "cell_type": "markdown", + "id": "b5f0b66f-36c0-48ab-8da9-817414ba6083", + "metadata": {}, + "source": [ + "That's all! Let's take a closer look at what this has actually done." + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "54ff5947-3c4a-4030-8714-fc392810b1d2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
smilesmy_propertymolecule
0CN1C=NC2=C1C(=O)N(C)C(=O)N2Cmy_value/home/cas/.cache/polaris-tutorials/003/data.za...
\n", + "
" + ], + "text/plain": [ + " smiles my_property \\\n", + "0 CN1C=NC2=C1C(=O)N(C)C(=O)N2C my_value \n", + "\n", + " molecule \n", + "0 /home/cas/.cache/polaris-tutorials/003/data.za... " + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "c" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "34022d65-7d1f-41ca-902d-a8385c4b6e40", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'smiles': ColumnAnnotation(is_pointer=False, modality=, description=None, user_attributes={}, dtype=dtype('O')),\n", + " 'my_property': ColumnAnnotation(is_pointer=False, modality=, description=None, user_attributes={}, dtype=dtype('O')),\n", + " 'molecule': ColumnAnnotation(is_pointer=True, modality=, description=None, user_attributes={}, dtype=dtype('O'))}" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.annotations" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "451b687e-34dd-4a86-9d36-b39d6247a24e", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZI0lEQVR4nO3de1hU1d4H8O8w3DHlYggaat4V0YRTjyhlgiAaCCKopGLlm+Xr8fR27Bw1FDnkBctL9qqlnkxMjEAkDK947PCGICcvKaAVISgoDigoSTAwM/v9Y0gDxYTZe7bK9/OXrplZ6zfPM8+Xvfdae22FIAggIqK2MpG7ACKiRxtjlIjIIIxRIiKDMEaJiAzCGCUiMghjlIjIIKZyF0BkdA0NOHwY589DEDBwIPz8YG4ud030CFNw3Si1LwUFCAhARQX+9CcAOHECDg7YuxcDB8pdGT2qGKPUnmg0GDoUdnbYvx8dOwJAdTXGj0d5OfLzYWYmd330SOK1UWpPDh7EuXNYt64xQwF07Ij161FQgLQ0WSujRxhjlNqT7Gx06oRnn23S6OEBBwdkZ8tUEz3yGKPUnlRWwtn5Hu3duuH6daNXQ48Jxii1J9bWqKy8R/v16+jQwejV0GOCMUrtiZsbKipQVtakUaVCWRnc3GSqiR55jFFqT4KC0KEDli9v0rhiBWxsEBIiU030yOPye2pP7OywdSumT4dKBV9fKBRIT0dKCnbsgL293MXRo4rrRql9+PZbqNUYMwYATp7Exo3Iy4MgYPBgzJ3buBSfqE0Yo9QOCAI8PHD6NBISMGWK3NXQ44bXRqkdSEnB6dNwdsaECQDQ0IDMTLlroscHY5Qed4KAmBgAWLwYVlYAEBeH55/HnDny1kWPDcYoyUMQhNraWmOMtHs3zpyBiwtmzQKAhgasWAEAL75ojNGpHWCMkgy+/vrrXr16RUVFST6STodlywAgMhIWFgDw6acoKoKrK8LCJB+d2gfGKMnAwcGhuLj4yy+/1Ol00o6UmIizZ9G9O159FQDq67FqFQBER8OEP34SB39JJANPT8+ePXuWlJRkZWVJOIxW23hVdMmSxo2Zt25FcTEGD+ZiexIRY5RkoFAoJk+eDCAhIUHCYRIScP48evZERAQAqNWIjQWAmBgeipKI+GMieUydOhVAYmKiRqORZACttvGq6NKljYeimzejtBTDhiE4WJIRqb1ijJI8hg0bNnDgwIqKim+++UaSAeLj8cMP6NMH06cDQF0d3n8fAKKjoVBIMiK1V4xRko105/UajUal73bJEpiaAsDHH+PyZbi7IzBQ9OGonWOMkmzCw8MB7NmzR61Wi9vzzp07nQ4cWB0QgGnTAKCuDqtXA0BMDA9FSXSMUZJN//79n3nmGVtbx3//+6KI3TY0NLz33nsAuoaHQ6kEUPPPf+LKFTz3HF56ScSBiPQYoySnV15JLy7+MS6un4h9xsXFXbhwoV+/flOmTAFQU1PTZ9my5c89V6ufcSISG2OU5DRxYmeFAqmpqKkRp8OGhoYVK1YAiImJUSqVADZs2HBVpdqnVFr5+oozBlFTjFGSU/fu8PTEr7+K9njjbdu2FRUVubq6hoWFAaipqVm7di2AGP06fCIJMEZJZvr9P0WZrq+vr4+NjQUQHR1tYmICYP369eXl5SNHjhyj37CZSALctplkdvUqnnoKpqa4ehW2tgZ1tWnTprlz5w4ePPjMmTMmJia3bt3q1atXRUXF0aNHR48eLVK9RM3xaJRk5uSEF1+EWo2vvjKoH7VavXLlSgAxMTH6Q9F169ZVVFR4eXkxQ0lSjFGSnyjn9Vu2bCktLR02bFhwcDCAmzdvfvjhhwCWcYKeJMYYJfmFhsLcHP/6F8rL29hDXV3dqlWrAPTr1y8iIuLq1avr1q2rrKz08fEZNWqUmLUS3YUPWCb52dlh+nTY2KANu5TU1tYeO3Zs9erVly9fBvDll1+amZmVlpaeOXMGgDF2hqZ2j1NMJL/SUmRn47nn0KNHY4tOh+RkjBiBbt3u/ZHCwuv79sXv27cvIyPj9r2knTt39vX1TUhIUCqVGo3Gz8/v0KFDRvkG1K7xaJTkl5ODyZPx3HPIzm7cCFSjweTJSElpEqNaLbKzkZaGI0dQXl5dUvIWABMTEw8PjzFjxgQEBIwcOVKhUFy8eFG/G/SSJUvk+T7UzjBG6aFgYoKiIvzzn5g9u/lLV69i/37s34/0dFRXNzba2T39+uv/8/zz7uPGjevcufPv3x8REZGVldW9e3cvLy+j1E7tHU/qSX7JyZg6FWvXIjoa58/D0RH19bCwQEoKdu3C7t24/SN1c8P48Rg/HiNGNG6Ad7eqqipnZ2eNRlNaWurk5GS0b0HtFmfq6WExZw6cnfHOO00au3WDpSXGjMGHH6K4GGfPIjYWL7zQYoYCsLOz8/f312q1iYmJUtdMBMYoPTxMTfHxx9i5E//6153GxYtx/TrS0/HWW3cmoP7QtGnTAMTHx0tQJlFzjFF6iDz/PF5+GfPmoaGhscXBAVZWre4nMDCwU6dOP/xQVlBwXdwKie7GGKWHy+rVuHIFmzcb1ImlpeWbb56tr78UH+8gUl1ELWKM0sPFyQkxMTD8Bs4xY7rX1WHnTnAOlaTGGKWHzty56NnT0E68vdGtGwoL8d13IpREdB+MUZKfoyN8fO78V6nEpk0YMwaOjm3v08SkcccTzjOR1LhulOS3YQMmTYKzs8jdnjoFDw84OuLy5fstkCIyEI9GSWYZGZg3Dx4ed2bnxeLuDldXlJc3WUFliLKysvT0dHH6oscIY5RktmgRAMydCzMz8TufOhUw+Ly+qKho/fr1vr6+3bt3DwkJqaurE6U2emzwVIfklJKC7Gw4OuIvf5Gk/5dfxvLlUCha/UFBEE6cOJGSkpKamnru3Dl9o5WVlY+PT2VlZdeuXUUulB5ljFGSjVYL/R5MUVF44glJhnByQkFBk22iGhpw6xZsbe+drQ0NyMjAv/+9Pi5udWlpqb7R3t4+ICAgKCjI39/f2tpakkLpUcYYJdns2IH8fDz9NF5/Xaoh3n4bW7bg888xfXpjy8GDmDABv/7a5Oao2locOYK0NHz1FcrLMWqUtrS01MXFZdy4cQEBAWPHjjU3N5eqRHr0MUZJHvX1eO89AFi2DJJmlKUl5s/HSy/Bzq75S+Xl2LsXqak4cgS3L3i6uWHs2Olr1oxyd3dXtOFyALU/jFGSx8aNKCqCm1vjLJB0XnwRV65g4cImN5jeuAE/P2RlQacDAKUSXl4IDkZwMHr3BuAIGLBmldoZxijJ4NYtxMYCQGxs43b30lEqsW4dfH0xcyZGjGhstLXFlSswN4eXFwICMHmy+KtWqf1gjJIMPvgA5eXw8sL48cYYztsboaGYMwcnT95pTElBr17o0MEYBdDjjetGydgqKrBuHYDGA1LjWLsWRUX43/+90zJkCDOUxMEYJWPbtGnvk0/WBwZi5EjjDdqtG5YuRUwMrl0z3qDUTvCeejKq4uLiAQMGCILy9OkLgwZ1kXq4N97A5ctISwMAjQbDhkGtRkFB8wVPRIbg0SgZVVRUlFqtDg8PM0KGNqN/SMnPPxt5WHr8MUbJePLy8uLj483NzaOiomQpwMsLM2bIMjI9zhijZDzvvvuuTqebM2dOr169jDCcIOC11xrP6G+Li4Mg8IyexMQYJSPJyclJS0vr0KHDIv2eTtJLSsLw4Zg1yzijUfvFGCUjWbhwoSAI8+fP79LFGFdFtVpERwOAp6cRRqN2jTP1ZAzx8fHTp0/v3LlzYWFhx44djTDitm2YNQt9++LcOW59T9Li74skdOnSpZSUlKSkpGPHjgGIjIw0TobW1zc+W/Qf/2CGkuR4NEri+/HHH/fs2ZOcnHzyd3dfOjo6Xrp0ycLCwggFfPQR3noLbm74/nvJ79kn4l9qEk9u7t7DhyO3b8/Ly9M3dOzY0d/fPyMjQ6VSrVq1yjgZWlODlSsBYPlyZigZA39lZLD8fERHY9AgDBnSLzU1Ly/Pzs5uxowZiYmJV65c8fPzq6ysfPLJJ2cYa8XmRx/h6lU8+ywCAowzILV3PBqlNhEEHD+O5GQkJ6O4uLHR0bGHh8fhJUtGjx5t+tslyRs3bjQ0NNTX1yuVSiPUdfMmVq8GgNjYtjyCiagNGKN0X9XV+OILnD0LrRb9+mHqVHTtiuJieHnh8uXG93TrhpAQTJoELy8rpdL3d5++efOmvb29mZnZzZs3z58/P3DgQKnr/eADVFbCxwfe3lIPRdSIU0zUstxcjB0LS0v4+MDcHJmZKCxEYiL8/eHiAlNTBAcjLAwjRjS7BllVVfX1118nJSWlp6er1Wp947JlyyIjIyWt99o19OqFX35BVhaXi5LxMEapBVotBg9Gly44dAj6qSH9zZUpKSgogFqNp55q/hGVCl99VX/wYKf9++vq6wEolcpRo0b179//448/HjZs2KlTpyQteenSwpiY3hMmIDVV0nGImmCMUguOHIGvb/PjOpUKPXrg/febPFe+ogIHDiApCQcPQqMB8OdnnvnexiYsLGzKlClOTk5qtdrR0bG6uvrnn3/u3bu3RPWWlJT069fP1fWFzz5Lc3Mzk2gUorvx2ii14ORJmJri2WebNHbpgt69G5/FUVTUOMWUkwP9H2NLS4wfj0mTPgoMNPndczgtLCxeeumlL774IiUl5Z133pGo3piYmLq6uv79OzNDyci44IlaUF0Ne/t73APUpQtu3gSA5cvxt7/h+HFYWiIgAHFxUKmQmoqICJO7nmUcEhICIDk5WaJiCwoKtm/frlQq5dqCj9ozHo1SCzp1wvXr0GiaJ6lKhe7dASA8HLW1mDQJ/v6wtr5/Z+PHj7exscnJySkpKXFxcRG92KioKI1G8/rrr/fv31/0zonuj0ej1AJ3d2i1yM9v0lhejsJCeHgAgI8P4uMREvKHGQrA2traz89PEIRUCWZ/cnNzExMTLS0tlyxZInrnRH+IMUotGD0affvi3Xf1s0aNoqNhYYHw8Db0N2nSJEhzXh8ZGanT6d58800pjnOJ/hBn6qllJ09i7Fg4O2PcOJibIyMDJ04gIQFBQW3o7JdffnF0dGxoaLhy5Yqjo6NYNf7nP/8ZPny4tbV1YWGhcXYyJWqGR6PUMg8P/PADZszA5csoKIC3N86da1uGAnjiiSe8vb21Wq245/WLFi0SBOHtt99mhpJceDRKxrNt27ZZs2aNHTv24MGDonR45MgRX19fW1vbCxcu2N21PIDIOHg0SsYTFBRkamp69OjRyspKUTrUzyktWLCAGUoyYoyS8Tg4OIwaNaqhoSGt2eM6W+/06dMRERHHjx93cnKaN2+eKOURtQ1jlIzKwPn6/Pz86OjoAQMGuLu7f/755507dw4MDLSxsRG1RqLW4fJ7MqqQkJB58+YdOnSourr6AZ/LJAhCTk7O7t27k5OTi3/b29TZ2bl3796ZmZnWD7BqlUhSPBolo+rSpYunp6darT5w4MD936nT6U6ePBkdHd23b19PT881a9YUFxc/9dRTs2fP3rt376VLl958800AKpXKKIUTtYhHo2RsISEhmZmZycnJU6ZMuftVnU6XlZWVlJSUnJx8+bedoV1cXCZOnBgWFjZixAiT3/Y21a9wKi8vN1rlRPfEBU9kbCUlJT169LCysqqoqLh9Sq7VarOzs5OSkpKSksrKyvSNPXr0CAoKCgsLGzlypOKuR4Lk5uYOGTLE1dX19hP0iGTBo1EyNhcXFw8PjxMnThw+fDgwMFCfnomJiVevXtW/oWfPnhMmTLhnel67dq1z5876f+uPRnlST7Lj0SjJIDY2dtGiRX369KmsrLy9hnTAgAGhoaGhoaFDhw5t9v6SkpI9e/YkJSXl5ORcvHixa9euAHQ6nYWFhVarVavVZmbcY5Rkw6NRksELL7zg7OxcWFgoCMKgQYMCAwMDAgK8vLyave3ChQvJycm7d+/+7rvv9H/vra2tz549q49RExMTBwcHlUp17do1Z2dnGb4GEQDGKMkiOzu7rKzM3d09Pj5+wIABzV4tLi5OTU1NSkrKysrSp6eVlZWPj09YWNjEiROfeOKJ2+/s0qWLSqVSqVSMUZIRY5RksGPHDgBRUVG/z9ALFy7onyd67NgxfYu1tbW3t3dYWFhISEiHDh3u7ke/UxQvj5K8GKNkbKdPnz579qyDg8O4ceP0LcnJyUuXLs3/bYtoOzu7wMDA0NBQPz8/C/1DSVswZMgKlSq6qmqw5EUTtYwxSsYWFxcHYNq0aebm5voWhUKRn59vZ2cXEBAQFhY2duzY2y/9kWdzc1FaKlmtRA+AMUpGpdFoEhISAMycOfN2o7+//6FDh7y9vU3vfoLefem3GOU5PcmLMUpGtW/fPpVK5erq6u7ufrtR/6SmNvSm30Sf9zGRvHhPPRmV/oz+lVdeEaU3Ho3Sw4AxSsZTWVm5f/9+U1PTadOmidIhY5QeBoxRMp5du3ap1Wo/Pz+xlnnqT+oZoyQvxigZj/6M/veTSwZydIRCgYoK6HRidUnUarynnozk3Llzrq6unTp1Kisrs7KyEqtbe3tUVeHaNTg4iNUlUetwpp6MZNeuviNHFo0alS1ihgLo0gVVVVCpGKMkGx6NkjHodOjRA6WlyMqCp6eYPR89CqUSf/oT+EAmkguPRskY0tNRWoq+fTF8uDgd1tUhIgKurli69E7jP/6Bbt3wX/8lzhBED4hTTGQMcXEAMHMm7trDvo00GiQlIToaR47caczIwKlT4vRP9OAYoyS56mqkpkKhwMsvi9yzpyfmzoVaLXK3RK3CGCXJJSbi118xejSeflrknv/2N1RXIzZW5G6JWoUxSpK7fUYvuo4dsXIlYmNRUCB+50QPiDFK0ioqwrFjsLFBSIgk/c+cCQ8P/Pd/S9I50YNgjJK0tm+HICA0FPfavb4Vbt68d7tCgU8+QUYG9uwxqH+iNmOMkoQEATt3Agaf0efno29fbN5871cHD8af/4y//x319QaNQtQ2jFGS0P/9Hy5cQI8eGDWq7Z0UF8PPDxUVOHwYLd0sEh2N2lpkZ7d9FKI2Y4yShFJTAWDGDJi09Yd27Rr8/XHlCl58EfHxLS477dgRa9ZwgxKSB+9iIgmtXo2AAPTt28aP//IL/P3x448YMgQpKbC0BIC0NOzejU2bsHkzfv9s5qlT0dAg/poqoj/Ee+pJTIGBUKuxd29j5AF4/30UFGDr1lZ3VV+PwEAcPozevZGZCScnAMjMhJ8famuxfbskK6iI2oAn9SSm3Fykp2PlyjstJSX46adW96PTYfp0HD6Mrl2Rnt6YoXl5mDABtbWYPZsZSg8RxiiJbNw4xMbi/HmDOlm6tCApCba2OHCg8Ty9qAh+fqiqQlAQNm4UpVIicTBGSWT+/vDxwZw5Lc6q/6HFixcvXz5gzJj/fP01hgwBgIoKjBuHsjKMHo2EBLTyMcxE0mKMkvjWrkV2duOK0dbatGnT8uXLTUwUc+aUenkBQHV140TT0KHYs+fOVVeihwRjlMQ3YADmz8c776CqqnUf3LVr17x58xQKxZYtW0JCQgDU1yM0FKdOoU8fHDoEW1tJCiYyBGOUJLF4MWxsEBPTio8cOXLk1Vdf1el0H3zwwWuvvQZAq8W0aUhPb5xo0j9OmehhwxglcVRWNvmvtTXWrsXGjXem6e+/Nj4nJyc4OLi+vn7hwoXz588HIAhCVNT+r76CnR0OHULPntLUTWQwxigZqrYWCxdi4EBcvdqkPTgYfn44fBgAyssxaBASEu7dQ35+/vjx42tqaiIiIlasWKFvjIyMXLHipeHDV6WlYfBgSb8BkUEYo2SQzEy4uWHVKty4gays5q9u3AhrawDYtAk//ojwcMyejZqa5m9LTk6urKwMDg7etm2bQqEAsGHDhpUrV5qZmS1a5DZihBG+B5EBBKI2qa0VFiwQlEoBEAYPFk6dEgRB+PZb4dKlJm87fVr47jtBpxM2bxasrQVAePpp4dix5r3t2LGjtrZW/++dO3eamJgoFIrPPvtM+u9BZCjGKLXFmTPC0KECIJiaCgsWCHV1D/Sp3FxhyBABEMzMhDVrqjQazd3vSUtLMzMzA7BmzRqRiyaSBmOUWqehQYiNFczNBUDo3Vv49tvWfbyuTliwQDA11Q4b5uPp6VlYWPj7V48fP25jYwMgMjJSzKKJpMQYpVbIyxM8PARAUCiE2bOFW7fa2M8331xwdnYGYGtr+8UXX+gbc3Nz7e3tAcycOVOn04lWNJHEGKP0QDQaTWxs7JAhtwChVy8hI8PQDisqKoKDg/UX6MPCwnJzc11cXABMmDChoaFBjJKJjIQb5dEf++mnn1555ZXs7Oy+fYPHjNnz/vsKAx+sdNuWLVv++te/1tTUKJVKrVY7evTo/fv3W/J+T3qkMEbpfgRB2Lp1qz7pnJyctm7dGhAQIO4QRUVF4eHh+fn5Dg4O33//vS3v96RHDWOUWnTx4sXXXnvt6NGjAMLCwj755BP9tUvR3bp16+LFi3369LGwsJCifyJJMUbp3pKSkt54442qqipHR8dPPvlk4sSJcldE9JDiXUzUnEqlCgoKmjx5clVVVWhoaF5eHjOU6D64/y01kZmZGRQUVFlZaW9vv2HDhvDwcLkrInrY8aSemrhx44abm5urq+unn37arVs3ucshegQwRqm5kpIS/RJOInoQjFEiIoNwiomIyCCMUSIigzBGiYgMwhglIjIIY5SIyCD/D9+G3rhq5bBLAAABTnpUWHRyZGtpdFBLTCByZGtpdCAyMDIzLjA5LjUAAHice79v7T0GIOBnQAA+KL+BkY0hAyTAyMzOoAFiMEMFmBkRAmCaBZ3mgNBMaBoZmQkq4GZgZGBkYmBi5mBiZmFgYeVgYmVjYGPnYGLjYODgZODgYuDi5mDi4mHg4WVgZWTgYWEQYQJqZGUEKmdlY+Pg4mFhFd8EMgqKGfiWv+A4EMzqfeAh9+T9qasm7JdQkz+wae76fb+tPuxjNbE9sOuWlf2P4MN2sicZDxgfmWl/TnKiXfyMnP0T627b/arT2h/tNG//60ds+3u8qvY36/fse1i1Z/+O9a/3u/zi3a/3X/RA1r2N+5oDM+2ntG8Fmm+w//3Jz/Y6V6QOeL8SsZcsnm5vzfh2n334ZPt9B4Udls1+su+DWIZ93K5OuwWdH+yuhX2xf28hat9UvM9eDACM1GEAYrR3BQAAAaF6VFh0TU9MIHJka2l0IDIwMjMuMDkuNQAAeJx9U0tOxDAM3fcUvgCR7TgfL5kZhBCiI8HAHdhzf2EnHZJsaOsqcZ+d52d3A7/eL6/fP/B3xcu2AeA/j6rCV0TE7Q18Aaen55cdzrfH091zvn7utw8gAUoWY/eKfbxd3+4egjM8cKCojAgPMVDhtqJAUitOwQx7c0dsAA5Sc/EVhijOaECjZ8UgWQ+ophwdEAqrzkixpBZfKXUgcvHvFBgpzsBkKTFwjdKPVBZuGYVrmYG5n51zxl4Hx3zEJM7L4aVDtZSjDi6x9iApnGZohau7qzRWnitJ7pSt+IWAuk6uSZFGUInkSEqVZyThwVWRWtGK96RCukKp1Z8qlYakGrkTkcxLUcRG1QCJPKfRw3pUJ5KXmig28RVdKhfX1WmSZl06b5N0buxqO8lYcG+S1LKc/bRflunq83a67pcxb37zmCnbQBxzYxuQMRxklsYIsFkejSazMprJZnU0jGyroynkNkvfHDQJzP4inoSk5omTYNReMinjJEZM6og0KzLX7/v7v2rr7RdbH7+0RVgL8gAAAOF6VFh0U01JTEVTIHJka2l0IDIwMjMuMDkuNQAAeJwlT0uuxDAIu8pbtlIaBUP4qOoq+86F5vAPMiuC7dhmvbSed+FZtI7nc77HOn8PrL/vcaETB9rFnQw5qZO43zV55I4urtau0Vn4ziEaG46p3EY3RNxJOs1CBywadQziROEs9TUgSK3ArSxUtYLAutkJjYLDbAfB2IsXw6wiLsRbN0UrI4tsGxaTdA0i2XJybPcYlBVi/NRCgawynSxRckbbVyDuXCdRS8fhO1lEZ2pjZMHsodH2vX6Xndc2HXWfuMX5/QcEX0W59Lht5AAAAABJRU5ErkJggg==", + "text/plain": [ + "" + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.get_data(row=0, col=\"molecule\")" + ] + }, + { + "cell_type": "markdown", + "id": "6c822bda-1fe1-4c9e-8437-4714a6855baf", + "metadata": {}, + "source": [ + "We can see that Polaris has: \n", + "- Saved the molecule in an external Zarr archive and set the column annotations accordingly.\n", + "- Has extracted the molecule-level properties as additional columns.\n", + "- Has added an additional column with the SMILES.\n", + "- Effortlessly saves and loads the molecule object from the Zarr." + ] + }, + { + "cell_type": "markdown", + "id": "8913a820-3649-4566-8049-195480070d9c", + "metadata": {}, + "source": [ + "## Factory Design Pattern\n", + "If you've been dilligently going through the tutorials, you might remember that there is a function that seems to be doing something similar. And you would be right!" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "18beb7e0-95f2-4fd2-917d-8d4bcceb65af", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZI0lEQVR4nO3de1hU1d4H8O8w3DHlYggaat4V0YRTjyhlgiAaCCKopGLlm+Xr8fR27Bw1FDnkBctL9qqlnkxMjEAkDK947PCGICcvKaAVISgoDigoSTAwM/v9Y0gDxYTZe7bK9/OXrplZ6zfPM8+Xvfdae22FIAggIqK2MpG7ACKiRxtjlIjIIIxRIiKDMEaJiAzCGCUiMghjlIjIIKZyF0BkdA0NOHwY589DEDBwIPz8YG4ud030CFNw3Si1LwUFCAhARQX+9CcAOHECDg7YuxcDB8pdGT2qGKPUnmg0GDoUdnbYvx8dOwJAdTXGj0d5OfLzYWYmd330SOK1UWpPDh7EuXNYt64xQwF07Ij161FQgLQ0WSujRxhjlNqT7Gx06oRnn23S6OEBBwdkZ8tUEz3yGKPUnlRWwtn5Hu3duuH6daNXQ48Jxii1J9bWqKy8R/v16+jQwejV0GOCMUrtiZsbKipQVtakUaVCWRnc3GSqiR55jFFqT4KC0KEDli9v0rhiBWxsEBIiU030yOPye2pP7OywdSumT4dKBV9fKBRIT0dKCnbsgL293MXRo4rrRql9+PZbqNUYMwYATp7Exo3Iy4MgYPBgzJ3buBSfqE0Yo9QOCAI8PHD6NBISMGWK3NXQ44bXRqkdSEnB6dNwdsaECQDQ0IDMTLlroscHY5Qed4KAmBgAWLwYVlYAEBeH55/HnDny1kWPDcYoyUMQhNraWmOMtHs3zpyBiwtmzQKAhgasWAEAL75ojNGpHWCMkgy+/vrrXr16RUVFST6STodlywAgMhIWFgDw6acoKoKrK8LCJB+d2gfGKMnAwcGhuLj4yy+/1Ol00o6UmIizZ9G9O159FQDq67FqFQBER8OEP34SB39JJANPT8+ePXuWlJRkZWVJOIxW23hVdMmSxo2Zt25FcTEGD+ZiexIRY5RkoFAoJk+eDCAhIUHCYRIScP48evZERAQAqNWIjQWAmBgeipKI+GMieUydOhVAYmKiRqORZACttvGq6NKljYeimzejtBTDhiE4WJIRqb1ijJI8hg0bNnDgwIqKim+++UaSAeLj8cMP6NMH06cDQF0d3n8fAKKjoVBIMiK1V4xRko105/UajUal73bJEpiaAsDHH+PyZbi7IzBQ9OGonWOMkmzCw8MB7NmzR61Wi9vzzp07nQ4cWB0QgGnTAKCuDqtXA0BMDA9FSXSMUZJN//79n3nmGVtbx3//+6KI3TY0NLz33nsAuoaHQ6kEUPPPf+LKFTz3HF56ScSBiPQYoySnV15JLy7+MS6un4h9xsXFXbhwoV+/flOmTAFQU1PTZ9my5c89V6ufcSISG2OU5DRxYmeFAqmpqKkRp8OGhoYVK1YAiImJUSqVADZs2HBVpdqnVFr5+oozBlFTjFGSU/fu8PTEr7+K9njjbdu2FRUVubq6hoWFAaipqVm7di2AGP06fCIJMEZJZvr9P0WZrq+vr4+NjQUQHR1tYmICYP369eXl5SNHjhyj37CZSALctplkdvUqnnoKpqa4ehW2tgZ1tWnTprlz5w4ePPjMmTMmJia3bt3q1atXRUXF0aNHR48eLVK9RM3xaJRk5uSEF1+EWo2vvjKoH7VavXLlSgAxMTH6Q9F169ZVVFR4eXkxQ0lSjFGSnyjn9Vu2bCktLR02bFhwcDCAmzdvfvjhhwCWcYKeJMYYJfmFhsLcHP/6F8rL29hDXV3dqlWrAPTr1y8iIuLq1avr1q2rrKz08fEZNWqUmLUS3YUPWCb52dlh+nTY2KANu5TU1tYeO3Zs9erVly9fBvDll1+amZmVlpaeOXMGgDF2hqZ2j1NMJL/SUmRn47nn0KNHY4tOh+RkjBiBbt3u/ZHCwuv79sXv27cvIyPj9r2knTt39vX1TUhIUCqVGo3Gz8/v0KFDRvkG1K7xaJTkl5ODyZPx3HPIzm7cCFSjweTJSElpEqNaLbKzkZaGI0dQXl5dUvIWABMTEw8PjzFjxgQEBIwcOVKhUFy8eFG/G/SSJUvk+T7UzjBG6aFgYoKiIvzzn5g9u/lLV69i/37s34/0dFRXNzba2T39+uv/8/zz7uPGjevcufPv3x8REZGVldW9e3cvLy+j1E7tHU/qSX7JyZg6FWvXIjoa58/D0RH19bCwQEoKdu3C7t24/SN1c8P48Rg/HiNGNG6Ad7eqqipnZ2eNRlNaWurk5GS0b0HtFmfq6WExZw6cnfHOO00au3WDpSXGjMGHH6K4GGfPIjYWL7zQYoYCsLOz8/f312q1iYmJUtdMBMYoPTxMTfHxx9i5E//6153GxYtx/TrS0/HWW3cmoP7QtGnTAMTHx0tQJlFzjFF6iDz/PF5+GfPmoaGhscXBAVZWre4nMDCwU6dOP/xQVlBwXdwKie7GGKWHy+rVuHIFmzcb1ImlpeWbb56tr78UH+8gUl1ELWKM0sPFyQkxMTD8Bs4xY7rX1WHnTnAOlaTGGKWHzty56NnT0E68vdGtGwoL8d13IpREdB+MUZKfoyN8fO78V6nEpk0YMwaOjm3v08SkcccTzjOR1LhulOS3YQMmTYKzs8jdnjoFDw84OuLy5fstkCIyEI9GSWYZGZg3Dx4ed2bnxeLuDldXlJc3WUFliLKysvT0dHH6oscIY5RktmgRAMydCzMz8TufOhUw+Ly+qKho/fr1vr6+3bt3DwkJqaurE6U2emzwVIfklJKC7Gw4OuIvf5Gk/5dfxvLlUCha/UFBEE6cOJGSkpKamnru3Dl9o5WVlY+PT2VlZdeuXUUulB5ljFGSjVYL/R5MUVF44glJhnByQkFBk22iGhpw6xZsbe+drQ0NyMjAv/+9Pi5udWlpqb7R3t4+ICAgKCjI39/f2tpakkLpUcYYJdns2IH8fDz9NF5/Xaoh3n4bW7bg888xfXpjy8GDmDABv/7a5Oao2locOYK0NHz1FcrLMWqUtrS01MXFZdy4cQEBAWPHjjU3N5eqRHr0MUZJHvX1eO89AFi2DJJmlKUl5s/HSy/Bzq75S+Xl2LsXqak4cgS3L3i6uWHs2Olr1oxyd3dXtOFyALU/jFGSx8aNKCqCm1vjLJB0XnwRV65g4cImN5jeuAE/P2RlQacDAKUSXl4IDkZwMHr3BuAIGLBmldoZxijJ4NYtxMYCQGxs43b30lEqsW4dfH0xcyZGjGhstLXFlSswN4eXFwICMHmy+KtWqf1gjJIMPvgA5eXw8sL48cYYztsboaGYMwcnT95pTElBr17o0MEYBdDjjetGydgqKrBuHYDGA1LjWLsWRUX43/+90zJkCDOUxMEYJWPbtGnvk0/WBwZi5EjjDdqtG5YuRUwMrl0z3qDUTvCeejKq4uLiAQMGCILy9OkLgwZ1kXq4N97A5ctISwMAjQbDhkGtRkFB8wVPRIbg0SgZVVRUlFqtDg8PM0KGNqN/SMnPPxt5WHr8MUbJePLy8uLj483NzaOiomQpwMsLM2bIMjI9zhijZDzvvvuuTqebM2dOr169jDCcIOC11xrP6G+Li4Mg8IyexMQYJSPJyclJS0vr0KHDIv2eTtJLSsLw4Zg1yzijUfvFGCUjWbhwoSAI8+fP79LFGFdFtVpERwOAp6cRRqN2jTP1ZAzx8fHTp0/v3LlzYWFhx44djTDitm2YNQt9++LcOW59T9Li74skdOnSpZSUlKSkpGPHjgGIjIw0TobW1zc+W/Qf/2CGkuR4NEri+/HHH/fs2ZOcnHzyd3dfOjo6Xrp0ycLCwggFfPQR3noLbm74/nvJ79kn4l9qEk9u7t7DhyO3b8/Ly9M3dOzY0d/fPyMjQ6VSrVq1yjgZWlODlSsBYPlyZigZA39lZLD8fERHY9AgDBnSLzU1Ly/Pzs5uxowZiYmJV65c8fPzq6ysfPLJJ2cYa8XmRx/h6lU8+ywCAowzILV3PBqlNhEEHD+O5GQkJ6O4uLHR0bGHh8fhJUtGjx5t+tslyRs3bjQ0NNTX1yuVSiPUdfMmVq8GgNjYtjyCiagNGKN0X9XV+OILnD0LrRb9+mHqVHTtiuJieHnh8uXG93TrhpAQTJoELy8rpdL3d5++efOmvb29mZnZzZs3z58/P3DgQKnr/eADVFbCxwfe3lIPRdSIU0zUstxcjB0LS0v4+MDcHJmZKCxEYiL8/eHiAlNTBAcjLAwjRjS7BllVVfX1118nJSWlp6er1Wp947JlyyIjIyWt99o19OqFX35BVhaXi5LxMEapBVotBg9Gly44dAj6qSH9zZUpKSgogFqNp55q/hGVCl99VX/wYKf9++vq6wEolcpRo0b179//448/HjZs2KlTpyQteenSwpiY3hMmIDVV0nGImmCMUguOHIGvb/PjOpUKPXrg/febPFe+ogIHDiApCQcPQqMB8OdnnvnexiYsLGzKlClOTk5qtdrR0bG6uvrnn3/u3bu3RPWWlJT069fP1fWFzz5Lc3Mzk2gUorvx2ii14ORJmJri2WebNHbpgt69G5/FUVTUOMWUkwP9H2NLS4wfj0mTPgoMNPndczgtLCxeeumlL774IiUl5Z133pGo3piYmLq6uv79OzNDyci44IlaUF0Ne/t73APUpQtu3gSA5cvxt7/h+HFYWiIgAHFxUKmQmoqICJO7nmUcEhICIDk5WaJiCwoKtm/frlQq5dqCj9ozHo1SCzp1wvXr0GiaJ6lKhe7dASA8HLW1mDQJ/v6wtr5/Z+PHj7exscnJySkpKXFxcRG92KioKI1G8/rrr/fv31/0zonuj0ej1AJ3d2i1yM9v0lhejsJCeHgAgI8P4uMREvKHGQrA2traz89PEIRUCWZ/cnNzExMTLS0tlyxZInrnRH+IMUotGD0affvi3Xf1s0aNoqNhYYHw8Db0N2nSJEhzXh8ZGanT6d58800pjnOJ/hBn6qllJ09i7Fg4O2PcOJibIyMDJ04gIQFBQW3o7JdffnF0dGxoaLhy5Yqjo6NYNf7nP/8ZPny4tbV1YWGhcXYyJWqGR6PUMg8P/PADZszA5csoKIC3N86da1uGAnjiiSe8vb21Wq245/WLFi0SBOHtt99mhpJceDRKxrNt27ZZs2aNHTv24MGDonR45MgRX19fW1vbCxcu2N21PIDIOHg0SsYTFBRkamp69OjRyspKUTrUzyktWLCAGUoyYoyS8Tg4OIwaNaqhoSGt2eM6W+/06dMRERHHjx93cnKaN2+eKOURtQ1jlIzKwPn6/Pz86OjoAQMGuLu7f/755507dw4MDLSxsRG1RqLW4fJ7MqqQkJB58+YdOnSourr6AZ/LJAhCTk7O7t27k5OTi3/b29TZ2bl3796ZmZnWD7BqlUhSPBolo+rSpYunp6darT5w4MD936nT6U6ePBkdHd23b19PT881a9YUFxc/9dRTs2fP3rt376VLl958800AKpXKKIUTtYhHo2RsISEhmZmZycnJU6ZMuftVnU6XlZWVlJSUnJx8+bedoV1cXCZOnBgWFjZixAiT3/Y21a9wKi8vN1rlRPfEBU9kbCUlJT169LCysqqoqLh9Sq7VarOzs5OSkpKSksrKyvSNPXr0CAoKCgsLGzlypOKuR4Lk5uYOGTLE1dX19hP0iGTBo1EyNhcXFw8PjxMnThw+fDgwMFCfnomJiVevXtW/oWfPnhMmTLhnel67dq1z5876f+uPRnlST7Lj0SjJIDY2dtGiRX369KmsrLy9hnTAgAGhoaGhoaFDhw5t9v6SkpI9e/YkJSXl5ORcvHixa9euAHQ6nYWFhVarVavVZmbcY5Rkw6NRksELL7zg7OxcWFgoCMKgQYMCAwMDAgK8vLyave3ChQvJycm7d+/+7rvv9H/vra2tz549q49RExMTBwcHlUp17do1Z2dnGb4GEQDGKMkiOzu7rKzM3d09Pj5+wIABzV4tLi5OTU1NSkrKysrSp6eVlZWPj09YWNjEiROfeOKJ2+/s0qWLSqVSqVSMUZIRY5RksGPHDgBRUVG/z9ALFy7onyd67NgxfYu1tbW3t3dYWFhISEiHDh3u7ke/UxQvj5K8GKNkbKdPnz579qyDg8O4ceP0LcnJyUuXLs3/bYtoOzu7wMDA0NBQPz8/C/1DSVswZMgKlSq6qmqw5EUTtYwxSsYWFxcHYNq0aebm5voWhUKRn59vZ2cXEBAQFhY2duzY2y/9kWdzc1FaKlmtRA+AMUpGpdFoEhISAMycOfN2o7+//6FDh7y9vU3vfoLefem3GOU5PcmLMUpGtW/fPpVK5erq6u7ufrtR/6SmNvSm30Sf9zGRvHhPPRmV/oz+lVdeEaU3Ho3Sw4AxSsZTWVm5f/9+U1PTadOmidIhY5QeBoxRMp5du3ap1Wo/Pz+xlnnqT+oZoyQvxigZj/6M/veTSwZydIRCgYoK6HRidUnUarynnozk3Llzrq6unTp1Kisrs7KyEqtbe3tUVeHaNTg4iNUlUetwpp6MZNeuviNHFo0alS1ihgLo0gVVVVCpGKMkGx6NkjHodOjRA6WlyMqCp6eYPR89CqUSf/oT+EAmkguPRskY0tNRWoq+fTF8uDgd1tUhIgKurli69E7jP/6Bbt3wX/8lzhBED4hTTGQMcXEAMHMm7trDvo00GiQlIToaR47caczIwKlT4vRP9OAYoyS56mqkpkKhwMsvi9yzpyfmzoVaLXK3RK3CGCXJJSbi118xejSeflrknv/2N1RXIzZW5G6JWoUxSpK7fUYvuo4dsXIlYmNRUCB+50QPiDFK0ioqwrFjsLFBSIgk/c+cCQ8P/Pd/S9I50YNgjJK0tm+HICA0FPfavb4Vbt68d7tCgU8+QUYG9uwxqH+iNmOMkoQEATt3Agaf0efno29fbN5871cHD8af/4y//x319QaNQtQ2jFGS0P/9Hy5cQI8eGDWq7Z0UF8PPDxUVOHwYLd0sEh2N2lpkZ7d9FKI2Y4yShFJTAWDGDJi09Yd27Rr8/XHlCl58EfHxLS477dgRa9ZwgxKSB+9iIgmtXo2AAPTt28aP//IL/P3x448YMgQpKbC0BIC0NOzejU2bsHkzfv9s5qlT0dAg/poqoj/Ee+pJTIGBUKuxd29j5AF4/30UFGDr1lZ3VV+PwEAcPozevZGZCScnAMjMhJ8famuxfbskK6iI2oAn9SSm3Fykp2PlyjstJSX46adW96PTYfp0HD6Mrl2Rnt6YoXl5mDABtbWYPZsZSg8RxiiJbNw4xMbi/HmDOlm6tCApCba2OHCg8Ty9qAh+fqiqQlAQNm4UpVIicTBGSWT+/vDxwZw5Lc6q/6HFixcvXz5gzJj/fP01hgwBgIoKjBuHsjKMHo2EBLTyMcxE0mKMkvjWrkV2duOK0dbatGnT8uXLTUwUc+aUenkBQHV140TT0KHYs+fOVVeihwRjlMQ3YADmz8c776CqqnUf3LVr17x58xQKxZYtW0JCQgDU1yM0FKdOoU8fHDoEW1tJCiYyBGOUJLF4MWxsEBPTio8cOXLk1Vdf1el0H3zwwWuvvQZAq8W0aUhPb5xo0j9OmehhwxglcVRWNvmvtTXWrsXGjXem6e+/Nj4nJyc4OLi+vn7hwoXz588HIAhCVNT+r76CnR0OHULPntLUTWQwxigZqrYWCxdi4EBcvdqkPTgYfn44fBgAyssxaBASEu7dQ35+/vjx42tqaiIiIlasWKFvjIyMXLHipeHDV6WlYfBgSb8BkUEYo2SQzEy4uWHVKty4gays5q9u3AhrawDYtAk//ojwcMyejZqa5m9LTk6urKwMDg7etm2bQqEAsGHDhpUrV5qZmS1a5DZihBG+B5EBBKI2qa0VFiwQlEoBEAYPFk6dEgRB+PZb4dKlJm87fVr47jtBpxM2bxasrQVAePpp4dix5r3t2LGjtrZW/++dO3eamJgoFIrPPvtM+u9BZCjGKLXFmTPC0KECIJiaCgsWCHV1D/Sp3FxhyBABEMzMhDVrqjQazd3vSUtLMzMzA7BmzRqRiyaSBmOUWqehQYiNFczNBUDo3Vv49tvWfbyuTliwQDA11Q4b5uPp6VlYWPj7V48fP25jYwMgMjJSzKKJpMQYpVbIyxM8PARAUCiE2bOFW7fa2M8331xwdnYGYGtr+8UXX+gbc3Nz7e3tAcycOVOn04lWNJHEGKP0QDQaTWxs7JAhtwChVy8hI8PQDisqKoKDg/UX6MPCwnJzc11cXABMmDChoaFBjJKJjIQb5dEf++mnn1555ZXs7Oy+fYPHjNnz/vsKAx+sdNuWLVv++te/1tTUKJVKrVY7evTo/fv3W/J+T3qkMEbpfgRB2Lp1qz7pnJyctm7dGhAQIO4QRUVF4eHh+fn5Dg4O33//vS3v96RHDWOUWnTx4sXXXnvt6NGjAMLCwj755BP9tUvR3bp16+LFi3369LGwsJCifyJJMUbp3pKSkt54442qqipHR8dPPvlk4sSJcldE9JDiXUzUnEqlCgoKmjx5clVVVWhoaF5eHjOU6D64/y01kZmZGRQUVFlZaW9vv2HDhvDwcLkrInrY8aSemrhx44abm5urq+unn37arVs3ucshegQwRqm5kpIS/RJOInoQjFEiIoNwiomIyCCMUSIigzBGiYgMwhglIjIIY5SIyCD/D9+G3rhq5bBLAAABTnpUWHRyZGtpdFBLTCByZGtpdCAyMDIzLjA5LjUAAHice79v7T0GIOBnQAA+KL+BkY0hAyTAyMzOoAFiMEMFmBkRAmCaBZ3mgNBMaBoZmQkq4GZgZGBkYmBi5mBiZmFgYeVgYmVjYGPnYGLjYODgZODgYuDi5mDi4mHg4WVgZWTgYWEQYQJqZGUEKmdlY+Pg4mFhFd8EMgqKGfiWv+A4EMzqfeAh9+T9qasm7JdQkz+wae76fb+tPuxjNbE9sOuWlf2P4MN2sicZDxgfmWl/TnKiXfyMnP0T627b/arT2h/tNG//60ds+3u8qvY36/fse1i1Z/+O9a/3u/zi3a/3X/RA1r2N+5oDM+2ntG8Fmm+w//3Jz/Y6V6QOeL8SsZcsnm5vzfh2n334ZPt9B4Udls1+su+DWIZ93K5OuwWdH+yuhX2xf28hat9UvM9eDACM1GEAYrR3BQAAAaF6VFh0TU9MIHJka2l0IDIwMjMuMDkuNQAAeJx9U0tOxDAM3fcUvgCR7TgfL5kZhBCiI8HAHdhzf2EnHZJsaOsqcZ+d52d3A7/eL6/fP/B3xcu2AeA/j6rCV0TE7Q18Aaen55cdzrfH091zvn7utw8gAUoWY/eKfbxd3+4egjM8cKCojAgPMVDhtqJAUitOwQx7c0dsAA5Sc/EVhijOaECjZ8UgWQ+ophwdEAqrzkixpBZfKXUgcvHvFBgpzsBkKTFwjdKPVBZuGYVrmYG5n51zxl4Hx3zEJM7L4aVDtZSjDi6x9iApnGZohau7qzRWnitJ7pSt+IWAuk6uSZFGUInkSEqVZyThwVWRWtGK96RCukKp1Z8qlYakGrkTkcxLUcRG1QCJPKfRw3pUJ5KXmig28RVdKhfX1WmSZl06b5N0buxqO8lYcG+S1LKc/bRflunq83a67pcxb37zmCnbQBxzYxuQMRxklsYIsFkejSazMprJZnU0jGyroynkNkvfHDQJzP4inoSk5omTYNReMinjJEZM6og0KzLX7/v7v2rr7RdbH7+0RVgL8gAAAOF6VFh0U01JTEVTIHJka2l0IDIwMjMuMDkuNQAAeJwlT0uuxDAIu8pbtlIaBUP4qOoq+86F5vAPMiuC7dhmvbSed+FZtI7nc77HOn8PrL/vcaETB9rFnQw5qZO43zV55I4urtau0Vn4ziEaG46p3EY3RNxJOs1CBywadQziROEs9TUgSK3ArSxUtYLAutkJjYLDbAfB2IsXw6wiLsRbN0UrI4tsGxaTdA0i2XJybPcYlBVi/NRCgawynSxRckbbVyDuXCdRS8fhO1lEZ2pjZMHsodH2vX6Xndc2HXWfuMX5/QcEX0W59Lht5AAAAABJRU5ErkJggg==", + "text/plain": [ + "" + ] + }, + "execution_count": 59, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from polaris.dataset import create_dataset_from_file\n", + "\n", + "dataset = create_dataset_from_file(path, save_dst)\n", + "dataset.get_data(row=0, col=\"molecule\")" + ] + }, + { + "cell_type": "markdown", + "id": "f31f61de-3818-4b52-9548-a8f8d2cc752d", + "metadata": {}, + "source": [ + "The `DatasetFactory` is based on the factory design pattern. That way, you can easily create and add your own file converters. However, the defaults are set to be a good option for most people. \n", + "\n", + "Let's consider two cases that show the power of the `DatasetFactory` design. \n", + "\n", + "### Configuring the converter\n", + "Let's assume we do not want to extract the properties as separate columns, but rather keep them in the RDKit object. We cannot do this with the default converter, but we can configure its behavior to achieve this. " + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "35b6e2cb-3b45-4944-903d-7da81ff1e7a4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2024-03-14 15:26:05.284\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m99\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n" + ] + } + ], + "source": [ + "save_dst = dm.fs.join(SAVE_DIR, \"data2.zarr\")\n", + "factory.reset(save_dst)\n", + "\n", + "# Configure the converter\n", + "converter = SDFConverter(mol_prop_as_cols=False)\n", + "\n", + "# Overwrite the converter for SDF files\n", + "factory.register_converter(\"sdf\", converter)\n", + "\n", + "# Process the SDF file again\n", + "factory.add_from_file(path)\n", + "\n", + "# Build the dataset\n", + "dataset = factory.build()" + ] + }, + { + "cell_type": "markdown", + "id": "d066c9b5-2c5c-471c-8739-c63f1eca8b54", + "metadata": {}, + "source": [ + "And voila! The property is saved to the Zarr instead of to a separate column. " + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "dbd94922-a9b9-4096-b42b-4e593581b947", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZI0lEQVR4nO3de1hU1d4H8O8w3DHlYggaat4V0YRTjyhlgiAaCCKopGLlm+Xr8fR27Bw1FDnkBctL9qqlnkxMjEAkDK947PCGICcvKaAVISgoDigoSTAwM/v9Y0gDxYTZe7bK9/OXrplZ6zfPM8+Xvfdae22FIAggIqK2MpG7ACKiRxtjlIjIIIxRIiKDMEaJiAzCGCUiMghjlIjIIKZyF0BkdA0NOHwY589DEDBwIPz8YG4ud030CFNw3Si1LwUFCAhARQX+9CcAOHECDg7YuxcDB8pdGT2qGKPUnmg0GDoUdnbYvx8dOwJAdTXGj0d5OfLzYWYmd330SOK1UWpPDh7EuXNYt64xQwF07Ij161FQgLQ0WSujRxhjlNqT7Gx06oRnn23S6OEBBwdkZ8tUEz3yGKPUnlRWwtn5Hu3duuH6daNXQ48Jxii1J9bWqKy8R/v16+jQwejV0GOCMUrtiZsbKipQVtakUaVCWRnc3GSqiR55jFFqT4KC0KEDli9v0rhiBWxsEBIiU030yOPye2pP7OywdSumT4dKBV9fKBRIT0dKCnbsgL293MXRo4rrRql9+PZbqNUYMwYATp7Exo3Iy4MgYPBgzJ3buBSfqE0Yo9QOCAI8PHD6NBISMGWK3NXQ44bXRqkdSEnB6dNwdsaECQDQ0IDMTLlroscHY5Qed4KAmBgAWLwYVlYAEBeH55/HnDny1kWPDcYoyUMQhNraWmOMtHs3zpyBiwtmzQKAhgasWAEAL75ojNGpHWCMkgy+/vrrXr16RUVFST6STodlywAgMhIWFgDw6acoKoKrK8LCJB+d2gfGKMnAwcGhuLj4yy+/1Ol00o6UmIizZ9G9O159FQDq67FqFQBER8OEP34SB39JJANPT8+ePXuWlJRkZWVJOIxW23hVdMmSxo2Zt25FcTEGD+ZiexIRY5RkoFAoJk+eDCAhIUHCYRIScP48evZERAQAqNWIjQWAmBgeipKI+GMieUydOhVAYmKiRqORZACttvGq6NKljYeimzejtBTDhiE4WJIRqb1ijJI8hg0bNnDgwIqKim+++UaSAeLj8cMP6NMH06cDQF0d3n8fAKKjoVBIMiK1V4xRko105/UajUal73bJEpiaAsDHH+PyZbi7IzBQ9OGonWOMkmzCw8MB7NmzR61Wi9vzzp07nQ4cWB0QgGnTAKCuDqtXA0BMDA9FSXSMUZJN//79n3nmGVtbx3//+6KI3TY0NLz33nsAuoaHQ6kEUPPPf+LKFTz3HF56ScSBiPQYoySnV15JLy7+MS6un4h9xsXFXbhwoV+/flOmTAFQU1PTZ9my5c89V6ufcSISG2OU5DRxYmeFAqmpqKkRp8OGhoYVK1YAiImJUSqVADZs2HBVpdqnVFr5+oozBlFTjFGSU/fu8PTEr7+K9njjbdu2FRUVubq6hoWFAaipqVm7di2AGP06fCIJMEZJZvr9P0WZrq+vr4+NjQUQHR1tYmICYP369eXl5SNHjhyj37CZSALctplkdvUqnnoKpqa4ehW2tgZ1tWnTprlz5w4ePPjMmTMmJia3bt3q1atXRUXF0aNHR48eLVK9RM3xaJRk5uSEF1+EWo2vvjKoH7VavXLlSgAxMTH6Q9F169ZVVFR4eXkxQ0lSjFGSnyjn9Vu2bCktLR02bFhwcDCAmzdvfvjhhwCWcYKeJMYYJfmFhsLcHP/6F8rL29hDXV3dqlWrAPTr1y8iIuLq1avr1q2rrKz08fEZNWqUmLUS3YUPWCb52dlh+nTY2KANu5TU1tYeO3Zs9erVly9fBvDll1+amZmVlpaeOXMGgDF2hqZ2j1NMJL/SUmRn47nn0KNHY4tOh+RkjBiBbt3u/ZHCwuv79sXv27cvIyPj9r2knTt39vX1TUhIUCqVGo3Gz8/v0KFDRvkG1K7xaJTkl5ODyZPx3HPIzm7cCFSjweTJSElpEqNaLbKzkZaGI0dQXl5dUvIWABMTEw8PjzFjxgQEBIwcOVKhUFy8eFG/G/SSJUvk+T7UzjBG6aFgYoKiIvzzn5g9u/lLV69i/37s34/0dFRXNzba2T39+uv/8/zz7uPGjevcufPv3x8REZGVldW9e3cvLy+j1E7tHU/qSX7JyZg6FWvXIjoa58/D0RH19bCwQEoKdu3C7t24/SN1c8P48Rg/HiNGNG6Ad7eqqipnZ2eNRlNaWurk5GS0b0HtFmfq6WExZw6cnfHOO00au3WDpSXGjMGHH6K4GGfPIjYWL7zQYoYCsLOz8/f312q1iYmJUtdMBMYoPTxMTfHxx9i5E//6153GxYtx/TrS0/HWW3cmoP7QtGnTAMTHx0tQJlFzjFF6iDz/PF5+GfPmoaGhscXBAVZWre4nMDCwU6dOP/xQVlBwXdwKie7GGKWHy+rVuHIFmzcb1ImlpeWbb56tr78UH+8gUl1ELWKM0sPFyQkxMTD8Bs4xY7rX1WHnTnAOlaTGGKWHzty56NnT0E68vdGtGwoL8d13IpREdB+MUZKfoyN8fO78V6nEpk0YMwaOjm3v08SkcccTzjOR1LhulOS3YQMmTYKzs8jdnjoFDw84OuLy5fstkCIyEI9GSWYZGZg3Dx4ed2bnxeLuDldXlJc3WUFliLKysvT0dHH6oscIY5RktmgRAMydCzMz8TufOhUw+Ly+qKho/fr1vr6+3bt3DwkJqaurE6U2emzwVIfklJKC7Gw4OuIvf5Gk/5dfxvLlUCha/UFBEE6cOJGSkpKamnru3Dl9o5WVlY+PT2VlZdeuXUUulB5ljFGSjVYL/R5MUVF44glJhnByQkFBk22iGhpw6xZsbe+drQ0NyMjAv/+9Pi5udWlpqb7R3t4+ICAgKCjI39/f2tpakkLpUcYYJdns2IH8fDz9NF5/Xaoh3n4bW7bg888xfXpjy8GDmDABv/7a5Oao2locOYK0NHz1FcrLMWqUtrS01MXFZdy4cQEBAWPHjjU3N5eqRHr0MUZJHvX1eO89AFi2DJJmlKUl5s/HSy/Bzq75S+Xl2LsXqak4cgS3L3i6uWHs2Olr1oxyd3dXtOFyALU/jFGSx8aNKCqCm1vjLJB0XnwRV65g4cImN5jeuAE/P2RlQacDAKUSXl4IDkZwMHr3BuAIGLBmldoZxijJ4NYtxMYCQGxs43b30lEqsW4dfH0xcyZGjGhstLXFlSswN4eXFwICMHmy+KtWqf1gjJIMPvgA5eXw8sL48cYYztsboaGYMwcnT95pTElBr17o0MEYBdDjjetGydgqKrBuHYDGA1LjWLsWRUX43/+90zJkCDOUxMEYJWPbtGnvk0/WBwZi5EjjDdqtG5YuRUwMrl0z3qDUTvCeejKq4uLiAQMGCILy9OkLgwZ1kXq4N97A5ctISwMAjQbDhkGtRkFB8wVPRIbg0SgZVVRUlFqtDg8PM0KGNqN/SMnPPxt5WHr8MUbJePLy8uLj483NzaOiomQpwMsLM2bIMjI9zhijZDzvvvuuTqebM2dOr169jDCcIOC11xrP6G+Li4Mg8IyexMQYJSPJyclJS0vr0KHDIv2eTtJLSsLw4Zg1yzijUfvFGCUjWbhwoSAI8+fP79LFGFdFtVpERwOAp6cRRqN2jTP1ZAzx8fHTp0/v3LlzYWFhx44djTDitm2YNQt9++LcOW59T9Li74skdOnSpZSUlKSkpGPHjgGIjIw0TobW1zc+W/Qf/2CGkuR4NEri+/HHH/fs2ZOcnHzyd3dfOjo6Xrp0ycLCwggFfPQR3noLbm74/nvJ79kn4l9qEk9u7t7DhyO3b8/Ly9M3dOzY0d/fPyMjQ6VSrVq1yjgZWlODlSsBYPlyZigZA39lZLD8fERHY9AgDBnSLzU1Ly/Pzs5uxowZiYmJV65c8fPzq6ysfPLJJ2cYa8XmRx/h6lU8+ywCAowzILV3PBqlNhEEHD+O5GQkJ6O4uLHR0bGHh8fhJUtGjx5t+tslyRs3bjQ0NNTX1yuVSiPUdfMmVq8GgNjYtjyCiagNGKN0X9XV+OILnD0LrRb9+mHqVHTtiuJieHnh8uXG93TrhpAQTJoELy8rpdL3d5++efOmvb29mZnZzZs3z58/P3DgQKnr/eADVFbCxwfe3lIPRdSIU0zUstxcjB0LS0v4+MDcHJmZKCxEYiL8/eHiAlNTBAcjLAwjRjS7BllVVfX1118nJSWlp6er1Wp947JlyyIjIyWt99o19OqFX35BVhaXi5LxMEapBVotBg9Gly44dAj6qSH9zZUpKSgogFqNp55q/hGVCl99VX/wYKf9++vq6wEolcpRo0b179//448/HjZs2KlTpyQteenSwpiY3hMmIDVV0nGImmCMUguOHIGvb/PjOpUKPXrg/febPFe+ogIHDiApCQcPQqMB8OdnnvnexiYsLGzKlClOTk5qtdrR0bG6uvrnn3/u3bu3RPWWlJT069fP1fWFzz5Lc3Mzk2gUorvx2ii14ORJmJri2WebNHbpgt69G5/FUVTUOMWUkwP9H2NLS4wfj0mTPgoMNPndczgtLCxeeumlL774IiUl5Z133pGo3piYmLq6uv79OzNDyci44IlaUF0Ne/t73APUpQtu3gSA5cvxt7/h+HFYWiIgAHFxUKmQmoqICJO7nmUcEhICIDk5WaJiCwoKtm/frlQq5dqCj9ozHo1SCzp1wvXr0GiaJ6lKhe7dASA8HLW1mDQJ/v6wtr5/Z+PHj7exscnJySkpKXFxcRG92KioKI1G8/rrr/fv31/0zonuj0ej1AJ3d2i1yM9v0lhejsJCeHgAgI8P4uMREvKHGQrA2traz89PEIRUCWZ/cnNzExMTLS0tlyxZInrnRH+IMUotGD0affvi3Xf1s0aNoqNhYYHw8Db0N2nSJEhzXh8ZGanT6d58800pjnOJ/hBn6qllJ09i7Fg4O2PcOJibIyMDJ04gIQFBQW3o7JdffnF0dGxoaLhy5Yqjo6NYNf7nP/8ZPny4tbV1YWGhcXYyJWqGR6PUMg8P/PADZszA5csoKIC3N86da1uGAnjiiSe8vb21Wq245/WLFi0SBOHtt99mhpJceDRKxrNt27ZZs2aNHTv24MGDonR45MgRX19fW1vbCxcu2N21PIDIOHg0SsYTFBRkamp69OjRyspKUTrUzyktWLCAGUoyYoyS8Tg4OIwaNaqhoSGt2eM6W+/06dMRERHHjx93cnKaN2+eKOURtQ1jlIzKwPn6/Pz86OjoAQMGuLu7f/755507dw4MDLSxsRG1RqLW4fJ7MqqQkJB58+YdOnSourr6AZ/LJAhCTk7O7t27k5OTi3/b29TZ2bl3796ZmZnWD7BqlUhSPBolo+rSpYunp6darT5w4MD936nT6U6ePBkdHd23b19PT881a9YUFxc/9dRTs2fP3rt376VLl958800AKpXKKIUTtYhHo2RsISEhmZmZycnJU6ZMuftVnU6XlZWVlJSUnJx8+bedoV1cXCZOnBgWFjZixAiT3/Y21a9wKi8vN1rlRPfEBU9kbCUlJT169LCysqqoqLh9Sq7VarOzs5OSkpKSksrKyvSNPXr0CAoKCgsLGzlypOKuR4Lk5uYOGTLE1dX19hP0iGTBo1EyNhcXFw8PjxMnThw+fDgwMFCfnomJiVevXtW/oWfPnhMmTLhnel67dq1z5876f+uPRnlST7Lj0SjJIDY2dtGiRX369KmsrLy9hnTAgAGhoaGhoaFDhw5t9v6SkpI9e/YkJSXl5ORcvHixa9euAHQ6nYWFhVarVavVZmbcY5Rkw6NRksELL7zg7OxcWFgoCMKgQYMCAwMDAgK8vLyave3ChQvJycm7d+/+7rvv9H/vra2tz549q49RExMTBwcHlUp17do1Z2dnGb4GEQDGKMkiOzu7rKzM3d09Pj5+wIABzV4tLi5OTU1NSkrKysrSp6eVlZWPj09YWNjEiROfeOKJ2+/s0qWLSqVSqVSMUZIRY5RksGPHDgBRUVG/z9ALFy7onyd67NgxfYu1tbW3t3dYWFhISEiHDh3u7ke/UxQvj5K8GKNkbKdPnz579qyDg8O4ceP0LcnJyUuXLs3/bYtoOzu7wMDA0NBQPz8/C/1DSVswZMgKlSq6qmqw5EUTtYwxSsYWFxcHYNq0aebm5voWhUKRn59vZ2cXEBAQFhY2duzY2y/9kWdzc1FaKlmtRA+AMUpGpdFoEhISAMycOfN2o7+//6FDh7y9vU3vfoLefem3GOU5PcmLMUpGtW/fPpVK5erq6u7ufrtR/6SmNvSm30Sf9zGRvHhPPRmV/oz+lVdeEaU3Ho3Sw4AxSsZTWVm5f/9+U1PTadOmidIhY5QeBoxRMp5du3ap1Wo/Pz+xlnnqT+oZoyQvxigZj/6M/veTSwZydIRCgYoK6HRidUnUarynnozk3Llzrq6unTp1Kisrs7KyEqtbe3tUVeHaNTg4iNUlUetwpp6MZNeuviNHFo0alS1ihgLo0gVVVVCpGKMkGx6NkjHodOjRA6WlyMqCp6eYPR89CqUSf/oT+EAmkguPRskY0tNRWoq+fTF8uDgd1tUhIgKurli69E7jP/6Bbt3wX/8lzhBED4hTTGQMcXEAMHMm7trDvo00GiQlIToaR47caczIwKlT4vRP9OAYoyS56mqkpkKhwMsvi9yzpyfmzoVaLXK3RK3CGCXJJSbi118xejSeflrknv/2N1RXIzZW5G6JWoUxSpK7fUYvuo4dsXIlYmNRUCB+50QPiDFK0ioqwrFjsLFBSIgk/c+cCQ8P/Pd/S9I50YNgjJK0tm+HICA0FPfavb4Vbt68d7tCgU8+QUYG9uwxqH+iNmOMkoQEATt3Agaf0efno29fbN5871cHD8af/4y//x319QaNQtQ2jFGS0P/9Hy5cQI8eGDWq7Z0UF8PPDxUVOHwYLd0sEh2N2lpkZ7d9FKI2Y4yShFJTAWDGDJi09Yd27Rr8/XHlCl58EfHxLS477dgRa9ZwgxKSB+9iIgmtXo2AAPTt28aP//IL/P3x448YMgQpKbC0BIC0NOzejU2bsHkzfv9s5qlT0dAg/poqoj/Ee+pJTIGBUKuxd29j5AF4/30UFGDr1lZ3VV+PwEAcPozevZGZCScnAMjMhJ8famuxfbskK6iI2oAn9SSm3Fykp2PlyjstJSX46adW96PTYfp0HD6Mrl2Rnt6YoXl5mDABtbWYPZsZSg8RxiiJbNw4xMbi/HmDOlm6tCApCba2OHCg8Ty9qAh+fqiqQlAQNm4UpVIicTBGSWT+/vDxwZw5Lc6q/6HFixcvXz5gzJj/fP01hgwBgIoKjBuHsjKMHo2EBLTyMcxE0mKMkvjWrkV2duOK0dbatGnT8uXLTUwUc+aUenkBQHV140TT0KHYs+fOVVeihwRjlMQ3YADmz8c776CqqnUf3LVr17x58xQKxZYtW0JCQgDU1yM0FKdOoU8fHDoEW1tJCiYyBGOUJLF4MWxsEBPTio8cOXLk1Vdf1el0H3zwwWuvvQZAq8W0aUhPb5xo0j9OmehhwxglcVRWNvmvtTXWrsXGjXem6e+/Nj4nJyc4OLi+vn7hwoXz588HIAhCVNT+r76CnR0OHULPntLUTWQwxigZqrYWCxdi4EBcvdqkPTgYfn44fBgAyssxaBASEu7dQ35+/vjx42tqaiIiIlasWKFvjIyMXLHipeHDV6WlYfBgSb8BkUEYo2SQzEy4uWHVKty4gays5q9u3AhrawDYtAk//ojwcMyejZqa5m9LTk6urKwMDg7etm2bQqEAsGHDhpUrV5qZmS1a5DZihBG+B5EBBKI2qa0VFiwQlEoBEAYPFk6dEgRB+PZb4dKlJm87fVr47jtBpxM2bxasrQVAePpp4dix5r3t2LGjtrZW/++dO3eamJgoFIrPPvtM+u9BZCjGKLXFmTPC0KECIJiaCgsWCHV1D/Sp3FxhyBABEMzMhDVrqjQazd3vSUtLMzMzA7BmzRqRiyaSBmOUWqehQYiNFczNBUDo3Vv49tvWfbyuTliwQDA11Q4b5uPp6VlYWPj7V48fP25jYwMgMjJSzKKJpMQYpVbIyxM8PARAUCiE2bOFW7fa2M8331xwdnYGYGtr+8UXX+gbc3Nz7e3tAcycOVOn04lWNJHEGKP0QDQaTWxs7JAhtwChVy8hI8PQDisqKoKDg/UX6MPCwnJzc11cXABMmDChoaFBjJKJjIQb5dEf++mnn1555ZXs7Oy+fYPHjNnz/vsKAx+sdNuWLVv++te/1tTUKJVKrVY7evTo/fv3W/J+T3qkMEbpfgRB2Lp1qz7pnJyctm7dGhAQIO4QRUVF4eHh+fn5Dg4O33//vS3v96RHDWOUWnTx4sXXXnvt6NGjAMLCwj755BP9tUvR3bp16+LFi3369LGwsJCifyJJMUbp3pKSkt54442qqipHR8dPPvlk4sSJcldE9JDiXUzUnEqlCgoKmjx5clVVVWhoaF5eHjOU6D64/y01kZmZGRQUVFlZaW9vv2HDhvDwcLkrInrY8aSemrhx44abm5urq+unn37arVs3ucshegQwRqm5kpIS/RJOInoQjFEiIoNwiomIyCCMUSIigzBGiYgMwhglIjIIY5SIyCD/D9+G3rhq5bBLAAABTnpUWHRyZGtpdFBLTCByZGtpdCAyMDIzLjA5LjUAAHice79v7T0GIOBnQAA+KL+BkY0hAyTAyMzOoAFiMEMFmBkRAmCaBZ3mgNBMaBoZmQkq4GZgZGBkYmBi5mBiZmFgYeVgYmVjYGPnYGLjYODgZODgYuDi5mDi4mHg4WVgZWTgYWEQYQJqZGUEKmdlY+Pg4mFhFd8EMgqKGfiWv+A4EMzqfeAh9+T9qasm7JdQkz+wae76fb+tPuxjNbE9sOuWlf2P4MN2sicZDxgfmWl/TnKiXfyMnP0T627b/arT2h/tNG//60ds+3u8qvY36/fse1i1Z/+O9a/3u/zi3a/3X/RA1r2N+5oDM+2ntG8Fmm+w//3Jz/Y6V6QOeL8SsZcsnm5vzfh2n334ZPt9B4Udls1+su+DWIZ93K5OuwWdH+yuhX2xf28hat9UvM9eDACM1GEAYrR3BQAAAaF6VFh0TU9MIHJka2l0IDIwMjMuMDkuNQAAeJx9U0tOxDAM3fcUvgCR7TgfL5kZhBCiI8HAHdhzf2EnHZJsaOsqcZ+d52d3A7/eL6/fP/B3xcu2AeA/j6rCV0TE7Q18Aaen55cdzrfH091zvn7utw8gAUoWY/eKfbxd3+4egjM8cKCojAgPMVDhtqJAUitOwQx7c0dsAA5Sc/EVhijOaECjZ8UgWQ+ophwdEAqrzkixpBZfKXUgcvHvFBgpzsBkKTFwjdKPVBZuGYVrmYG5n51zxl4Hx3zEJM7L4aVDtZSjDi6x9iApnGZohau7qzRWnitJ7pSt+IWAuk6uSZFGUInkSEqVZyThwVWRWtGK96RCukKp1Z8qlYakGrkTkcxLUcRG1QCJPKfRw3pUJ5KXmig28RVdKhfX1WmSZl06b5N0buxqO8lYcG+S1LKc/bRflunq83a67pcxb37zmCnbQBxzYxuQMRxklsYIsFkejSazMprJZnU0jGyroynkNkvfHDQJzP4inoSk5omTYNReMinjJEZM6og0KzLX7/v7v2rr7RdbH7+0RVgL8gAAAOF6VFh0U01JTEVTIHJka2l0IDIwMjMuMDkuNQAAeJwlT0uuxDAIu8pbtlIaBUP4qOoq+86F5vAPMiuC7dhmvbSed+FZtI7nc77HOn8PrL/vcaETB9rFnQw5qZO43zV55I4urtau0Vn4ziEaG46p3EY3RNxJOs1CBywadQziROEs9TUgSK3ArSxUtYLAutkJjYLDbAfB2IsXw6wiLsRbN0UrI4tsGxaTdA0i2XJybPcYlBVi/NRCgawynSxRckbbVyDuXCdRS8fhO1lEZ2pjZMHsodH2vX6Xndc2HXWfuMX5/QcEX0W59Lht5AAAAABJRU5ErkJggg==", + "text/html": [ + "\n", + "
my_propertymy_value
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.get_data(row=0, col=\"molecule\")" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "2b12c7c0-23be-4286-8dca-23d0e7a606cf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
smilesmolecule
0CN1C=NC2=C1C(=O)N(C)C(=O)N2C/home/cas/.cache/polaris-tutorials/003/data2.z...
\n", + "
" + ], + "text/plain": [ + " smiles \\\n", + "0 CN1C=NC2=C1C(=O)N(C)C(=O)N2C \n", + "\n", + " molecule \n", + "0 /home/cas/.cache/polaris-tutorials/003/data2.z... " + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.table" + ] + }, + { + "cell_type": "markdown", + "id": "4db64c4d-4712-4dfe-81ae-c8daa01066de", + "metadata": {}, + "source": [ + "### Merging data from different sources\n", + "\n", + "Another case is when you want to merge data from multiple sources. Maybe you have two different SDF files." + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "ef15bf98-f301-465d-9e93-2531f9f1f98c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m2024-03-14 15:29:39.280\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m99\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n", + "\u001b[32m2024-03-14 15:29:39.284\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m99\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n" + ] + } + ], + "source": [ + "save_dst = dm.fs.join(SAVE_DIR, \"data3.zarr\")\n", + "factory.reset(save_dst)\n", + "\n", + "# Let's pretend these are two different SDF files\n", + "factory.register_converter(\"sdf\", SDFConverter(mol_column=\"molecule1\", smiles_column=None))\n", + "factory.add_from_file(path)\n", + "\n", + "# We change the configuration between files\n", + "factory.register_converter(\"sdf\", SDFConverter(mol_column=\"molecule2\", mol_prop_as_cols=False))\n", + "factory.add_from_file(path)\n", + "\n", + "dataset = factory.build()" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "65960c85-ee0d-4d37-b50d-b1c8ba1cec64", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
my_propertymolecule1smilesmolecule2
0my_value/home/cas/.cache/polaris-tutorials/003/data3.z...CN1C=NC2=C1C(=O)N(C)C(=O)N2C/home/cas/.cache/polaris-tutorials/003/data3.z...
\n", + "
" + ], + "text/plain": [ + " my_property molecule1 \\\n", + "0 my_value /home/cas/.cache/polaris-tutorials/003/data3.z... \n", + "\n", + " smiles \\\n", + "0 CN1C=NC2=C1C(=O)N(C)C(=O)N2C \n", + "\n", + " molecule2 \n", + "0 /home/cas/.cache/polaris-tutorials/003/data3.z... " + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.table" + ] + }, + { + "cell_type": "markdown", + "id": "a5d7bf37-7950-4026-b6c1-3fac556754ba", + "metadata": {}, + "source": [ + "The End. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/tutorials/dataset_zarr.ipynb b/docs/tutorials/dataset_zarr.ipynb index e4838c60..09211aa4 100644 --- a/docs/tutorials/dataset_zarr.ipynb +++ b/docs/tutorials/dataset_zarr.ipynb @@ -1,12 +1,36 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "217690be-9836-4e06-930e-ba7efbb37d91", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [ + "remove_cell" + ] + }, + "outputs": [], + "source": [ + "# Note: Cell is tagged to not show up in the mkdocs build\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, { "cell_type": "markdown", "id": "39b58e71", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ - "# Creating datasets with zarr\n", - "\n", "
\n", "

In short

\n", "

This tutorial shows how to create datasets with more advanced data-modalities through the .zarr format.

\n", @@ -22,7 +46,13 @@ { "cell_type": "markdown", "id": "e154bb54", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "### Dummy example\n", "For the sake of simplicity, let's assume we have just two datapoints. We will use this to demonstrate the idea behind pointer columns. " @@ -30,9 +60,15 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "5e201379", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ "import zarr\n", @@ -47,41 +83,90 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "07442028", - "metadata": {}, - "outputs": [], + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Create a single image and save it to a .zarr directory\n", - "images = np.random.random((2, 64, 64, 3))\n", - "\n", - "train_path = dm.fs.join(SAVE_DIR, \"single_train.zarr\")\n", - "zarr.save(train_path, images[0])\n", + "# Create two images and save them to a Zarr archive\n", + "base_path = dm.fs.join(SAVE_DIR, \"data.zarr\")\n", + "inp_col_name = \"images\"\n", "\n", - "test_path = dm.fs.join(SAVE_DIR, \"single_test.zarr\")\n", - "zarr.save(test_path, images[1])" + "images = np.random.random((2, 64, 64, 3))\n", + "root = zarr.open(base_path, \"w\")\n", + "root.array(inp_col_name, images)" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, + "id": "15df9619-e659-4558-9c69-416a186c1f3a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# For performance reasons, Polaris expects all data related to a column to be saved in a single Zarr array. \n", + "# To index a specific element in that array, the pointer path can have a suffix to specify the index. \n", + "train_path = f\"{base_path}/{inp_col_name}#0\"\n", + "test_path = f\"{base_path}/{inp_col_name}#1\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "16543db7", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ + "tgt_col_name = \"target\"\n", + "\n", "table = pd.DataFrame(\n", " {\n", - " \"images\": [train_path, test_path], # Instead of the content, we specify paths\n", - " \"target\": np.random.random(2),\n", + " inp_col_name: [train_path, test_path], # Instead of the content, we specify paths\n", + " tgt_col_name: np.random.random(2),\n", " }\n", ")" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "a257b09d", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ "from polaris.dataset import Dataset, ColumnAnnotation\n", @@ -97,24 +182,36 @@ { "cell_type": "markdown", "id": "2524c795", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "Note how the table does not contain the image data, but rather stores a path. " ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "id": "19a39fab", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { "text/plain": [ - "'/home/cas/.cache/polaris-tutorials/002/single_train.zarr'" + "'/home/cas/.cache/polaris-tutorials/002/data.zarr/images#0'" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -126,16 +223,28 @@ { "cell_type": "markdown", "id": "5c051877", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "To load the data that is being pointed to, you can simply use the `Dataset.get_data()` utility method. " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "8189f312", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "data": { @@ -143,7 +252,7 @@ "(64, 64, 3)" ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -155,24 +264,36 @@ { "cell_type": "markdown", "id": "17aaff10", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "Creating a benchmark and the associated `Subset` objects will automatically do so! " ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "6f1c8766", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [], "source": [ "from polaris.benchmark import SingleTaskBenchmarkSpecification\n", "\n", "benchmark = SingleTaskBenchmarkSpecification(\n", " dataset=dataset,\n", - " input_cols=\"images\",\n", - " target_cols=\"target\",\n", + " input_cols=inp_col_name,\n", + " target_cols=tgt_col_name,\n", " metrics=\"mean_absolute_error\",\n", " split=([0], [1]),\n", ")" @@ -180,9 +301,15 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "id": "9a0c635c", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "outputs": [ { "name": "stdout", @@ -203,130 +330,19 @@ { "cell_type": "markdown", "id": "67d2e77d", - "metadata": {}, - "source": [ - "## Creating datasets from `.zarr` arrays\n", - "\n", - "While the above example works, creating the table with all paths from scratch is time-consuming when datasets get large. Instead, you can also automatically parse a `.zarr` hierarchy into the expected tabular data structure. \n", - "\n", - "A little more about zarr: A `.zarr` file can contain groups and arrays, where each group can again contain groups and arrays. Each array can be saved as one or multiple chunks. Additional user attributes (for any array or group) are saved as JSON files.\n", - "\n", - "Within Polaris:\n", - "\n", - "1. Each subgroup of the root group corresponds to a single column.\n", - "2. Each subgroup can contain:\n", - " - A single array with all datapoints.\n", - " - A single array per datapoint.\n", - "3. Additional meta-data is saved to the user attributes of the root group.\n", - "4. The indices are required to be integers.\n", - "\n", - "To better explain how this works, let's look at two examples corresponding to the two cases in point 2 above. \n", - "\n", - "### A single array _per_ data point\n", - "In this first example we will create a zarr array _per_ data point. The structure of the zarr will look like: \n", - "\n", - "```\n", - "/\n", - " column_a/\n", - " array_1\n", - " array_2\n", - " ...\n", - " array_N\n", - "```\n", - "\n", - "and as we will see, this will get parsed into\n", - "\n", - "| column_a |\n", - "| ------------------------------------ |\n", - "| /path/to/root.zarr/column_a/array_1 |\n", - "| /path/to/root.zarr/column_a/array_2 |\n", - "| ... |\n", - "| /path/to/root.zarr/column_a/array_N |\n", - "\n", - "\n", - "
\n", - "

Note

\n", - "

Notice that the dataset now no longer stores the content of the array itself, but rather a reference to the array.

\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "04f47190", - "metadata": {}, - "outputs": [], - "source": [ - "# Let's first create some dummy dataset with 1000 64x64 \"images\"\n", - "images = np.random.random((1000, 64, 64, 3))" - ] - }, - { - "cell_type": "markdown", - "id": "d55e55f3", - "metadata": {}, - "source": [ - "To be able to use these images in Polaris, we need to save them in the zarr hierarchy." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "e4d4d32e", "metadata": { - "scrolled": true + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] }, - "outputs": [], "source": [ - "path = dm.fs.join(SAVE_DIR, \"zarr\", \"archive_multi.zarr\")\n", - "\n", - "with zarr.open(path, \"w\") as root:\n", - " with root.create_group(\"images\") as group:\n", - " for i, arr in enumerate(images):\n", - " # If you're saving an array per datapoint,\n", - " # the name of the array needs to be an integer\n", - " group.array(i, arr)\n", + "## Creating datasets from `.zarr` arrays\n", "\n", - " # The root directory can furthermore contain all additional meta-data in its user attributes.\n", - " root.attrs[\"name\"] = \"dummy_image_dataset\"\n", - " root.attrs[\"description\"] = \"Randomly generated 64x64 images\"\n", - " root.attrs[\"source\"] = \"https://doi.org/xx.xxxx\"\n", + "While the above example works, creating the table with all paths from scratch is time-consuming when datasets get large. Instead, you can also automatically parse a Zarr archive into the expected tabular data structure. \n", "\n", - " # To ensure proper processing, it is important that we annotate the column.\n", - " # As this has to be JSON serializable, we create a dict instead of the object.\n", - " # Due to using Pydantic, this will work seamlessly.\n", - " root.attrs[\"annotations\"] = {\"images\": {\"is_pointer\": True}}" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "f0885513", - "metadata": {}, - "outputs": [], - "source": [ - "dataset = Dataset.from_zarr(path)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "2a7809e1", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(64, 64, 3)" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset.get_data(col=\"images\", row=0).shape" + "A Zarr archive can contain groups and arrays, where each group can again contain groups and arrays. Within Polaris, we expect the root to be a flat hierarchy that contains a single array per column.\n" ] }, { @@ -335,13 +351,11 @@ "metadata": {}, "source": [ "### A single array for _all_ datapoints \n", - "Instead of having an array per datapoint, you might also batch all arrays in a single array. This could for example speed up compression.\n", "\n", - "In this case, our zarr hierarchy will look like this: \n", + "Polaris expects a flat zarr hierarchy, with a single array per pointer column: \n", "```\n", "/\n", - " column_a/\n", - " array\n", + " column_a\n", "```\n", "\n", "Which will get parsed into a table like: \n", @@ -361,45 +375,74 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, + "id": "622287ed-16ad-484e-a0d7-ca6cf648ed5d", + "metadata": {}, + "outputs": [], + "source": [ + "# Let's first create some dummy dataset with 1000 64x64 \"images\"\n", + "images = np.random.random((1000, 64, 64, 3))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, "id": "12a06b89", "metadata": {}, "outputs": [], "source": [ - "path = dm.fs.join(SAVE_DIR, \"zarr\", \"archive_single.zarr\")\n", + "path = dm.fs.join(SAVE_DIR, \"zarr\", \"data.zarr\")\n", "\n", "with zarr.open(path, \"w\") as root:\n", - " with root.create_group(\"images\") as group:\n", - " group.array(\"data\", images)" + " root.array(inp_col_name, images)" + ] + }, + { + "cell_type": "markdown", + "id": "59ddcf4b-6858-45d0-afd2-b396ee0bc498", + "metadata": {}, + "source": [ + "To create a dataset from a Zarr archive, we can use the convenience function `create_dataset_from_file()`." ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "3c7c11ac", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A\n" + ] + }, { "data": { "text/plain": [ - "'/home/cas/.cache/polaris-tutorials/002/zarr/archive_single.zarr//images/data#0'" + "'/home/cas/.cache/polaris-tutorials/002/zarr/data.zarr//images#0'" ] }, - "execution_count": 14, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dataset = Dataset.from_zarr(path)\n", + "from polaris.dataset import create_dataset_from_file\n", + "\n", + "# Because Polaris might restructure the Zarr archive, \n", + "# we need to specify a location to save the Zarr file to.\n", + "dataset = create_dataset_from_file(path, zarr_root_path=dm.fs.join(SAVE_DIR, \"zarr\", \"processed.zarr\"))\n", "\n", "# The path refers to the original zarr directory we created in the above code block\n", - "dataset.table.iloc[0][\"images\"]" + "dataset.table.iloc[0][inp_col_name]" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "f8d1b42d", "metadata": {}, "outputs": [ @@ -409,13 +452,13 @@ "(64, 64, 3)" ] }, - "execution_count": 15, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "dataset.get_data(col=\"images\", row=0).shape" + "dataset.get_data(col=inp_col_name, row=0).shape" ] }, { @@ -430,7 +473,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "1cd94077", "metadata": {}, "outputs": [], @@ -441,23 +484,19 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "id": "c5147684", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['/home/cas/.cache/polaris-tutorials/002/benchmark.json',\n", - " '/home/cas/.cache/polaris-tutorials/002/single_train.zarr',\n", - " '/home/cas/.cache/polaris-tutorials/002/dataset.json',\n", - " '/home/cas/.cache/polaris-tutorials/002/table.parquet',\n", - " '/home/cas/.cache/polaris-tutorials/002/zarr',\n", - " '/home/cas/.cache/polaris-tutorials/002/single_test.zarr',\n", - " '/home/cas/.cache/polaris-tutorials/002/json']" + "['/home/cas/.cache/polaris-tutorials/002/zarr',\n", + " '/home/cas/.cache/polaris-tutorials/002/json',\n", + " '/home/cas/.cache/polaris-tutorials/002/data.zarr']" ] }, - "execution_count": 17, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -472,18 +511,7 @@ "id": "b9bf6c19", "metadata": {}, "source": [ - "Besides the `table.parquet` and `dataset.yaml`, we can now also see a `data` folder which stores the content for the additional content from the pointer columns. Instead, we might want to rather save as a single `.zarr` file. With the `array_mode` argument, we can choose between the two structures we outlined in this repository. " - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "40e210fc", - "metadata": {}, - "outputs": [], - "source": [ - "savedir = dm.fs.join(SAVE_DIR, \"zarr\")\n", - "zarr_path = dataset.to_zarr(savedir, array_mode=\"single\")" + "Besides the `table.parquet` and `dataset.yaml`, we can now also see a `data` folder which stores the content for the additional content from the pointer columns." ] }, { @@ -496,28 +524,63 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 17, "id": "33c25a55", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
nameNone
description
tags
user_attributes
ownerNone
default_adaptersNone
md5sum6ef8d23737aafcbf82c421e7f99e1d95
readme
annotations
images
is_pointerTrue
modalityUNKNOWN
descriptionNone
user_attributes
dtypeobject
sourceNone
licenseNone
curation_referenceNone
cache_dir/home/cas/.cache/polaris/datasets/None/6ef8d23737aafcbf82c421e7f99e1d95
artifact_idNone
n_rows1000
n_columns1
" + ], + "text/plain": [ + "{\n", + " \"name\": null,\n", + " \"description\": \"\",\n", + " \"tags\": [],\n", + " \"user_attributes\": {},\n", + " \"owner\": null,\n", + " \"default_adapters\": null,\n", + " \"md5sum\": \"6ef8d23737aafcbf82c421e7f99e1d95\",\n", + " \"readme\": \"\",\n", + " \"annotations\": {\n", + " \"images\": {\n", + " \"is_pointer\": true,\n", + " \"modality\": \"UNKNOWN\",\n", + " \"description\": null,\n", + " \"user_attributes\": {},\n", + " \"dtype\": \"object\"\n", + " }\n", + " },\n", + " \"source\": null,\n", + " \"license\": null,\n", + " \"curation_reference\": null,\n", + " \"cache_dir\": \"/home/cas/.cache/polaris/datasets/None/6ef8d23737aafcbf82c421e7f99e1d95\",\n", + " \"artifact_id\": null,\n", + " \"n_rows\": 1000,\n", + " \"n_columns\": 1\n", + "}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "Dataset.from_json(json_path)" ] }, - { - "cell_type": "code", - "execution_count": 20, - "id": "6f7de196", - "metadata": {}, - "outputs": [], - "source": [ - "Dataset.from_zarr(zarr_path)" - ] - }, { "cell_type": "markdown", "id": "72767ef2", - "metadata": {}, + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, "source": [ "The End. " ] @@ -539,7 +602,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/mkdocs.yml b/mkdocs.yml index 28dc8e8e..a79cc9bb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -17,10 +17,12 @@ nav: - Polaris: index.md - Quickstart: quickstart.md - Tutorials: - - The Basics: tutorials/basics.ipynb - - Data Curation: tutorials/data_curation.ipynb - - Custom Datasets and Benchmarks: tutorials/custom_dataset_benchmark.ipynb - - Creating Datasets with zarr: tutorials/dataset_zarr.ipynb + - Basics: tutorials/basics.ipynb + - Data Models: tutorials/custom_dataset_benchmark.ipynb + - Creating Datasets: + - Data Curation: tutorials/data_curation.ipynb + - Zarr Datasets: tutorials/dataset_zarr.ipynb + - Dataset Factory: tutorials/dataset_factory.ipynb - API Reference: - Load: api/load.md - Core: diff --git a/polaris/dataset/__init__.py b/polaris/dataset/__init__.py index b09404d9..fbd2037f 100644 --- a/polaris/dataset/__init__.py +++ b/polaris/dataset/__init__.py @@ -1,6 +1,6 @@ from polaris.dataset._column import ColumnAnnotation, Modality from polaris.dataset._dataset import Dataset -from polaris.dataset._factory import DatasetFactory, get_dataset_from_file +from polaris.dataset._factory import DatasetFactory, create_dataset_from_file from polaris.dataset._subset import Subset __all__ = [ @@ -8,7 +8,6 @@ "Dataset", "Subset", "Modality", - "Adapter", "DatasetFactory", - "get_dataset_from_file", + "create_dataset_from_file", ] diff --git a/polaris/dataset/_adapters.py b/polaris/dataset/_adapters.py index 0d5a1008..6bf51ba5 100644 --- a/polaris/dataset/_adapters.py +++ b/polaris/dataset/_adapters.py @@ -1,70 +1,22 @@ -import abc -from typing import Any +from enum import Enum import datamol as dm -from pydantic import BaseModel -class Adapter(BaseModel, abc.ABC): +class Adapter(Enum): """ - Adapters are callable, serializable objects that can be used to _adapt_ the - datapoint in a dataset. This is for example - """ - - column: str - - def __call__(self, data: dict) -> dict: - """Adapts the entire datapoint - - Used like: - ```python - adapter = Adapter(column="my_column") - adapter({"my_column": datapoint}) - ``` - - Args: - data: The entire datapoint with column -> value pairs. - """ - if self.column not in data: - return data - v = data[self.column] - if isinstance(v, tuple): - data[self.column] = [self.adapt(x) for x in v] - else: - data[self.column] = self.adapt(v) - return data - - @abc.abstractmethod - def adapt(self, data: Any) -> Any: - """ - Adapt the value for a specific column. - This method has to be overwritten by subclasses. - - Used like: - ```python - adapter = Adapter(column="my_column") - adapter().adapt(datapoint["my_column"]) - ``` + Adapters are predefined callables that change the format of the data. + Adapters are serializable and can thus be saved alongside datasets. - Args: - data: The value to adapt - """ - raise NotImplementedError - - -class SmilesAdapter(Adapter): + Attributes: + SMILES_TO_MOL: Convert a SMILES string to a RDKit molecule. + BYTES_TO_MOL: Convert a RDKit binary string to a RDKit molecule. """ - Creates a RDKit `Mol` object from a SMILES string - """ - - def adapt(self, data: str) -> dm.Mol: - return dm.to_mol(data) + SMILES_TO_MOL = dm.to_mol + BYTES_TO_MOL = dm.Mol -class MolBytestringAdapter(Adapter): - """ - Creates a RDKit `Mol` object from the RDKit-specific bytestring serialization - """ - - def adapt(self, data: bytes) -> dm.Mol: - return dm.Mol(data) + def __call__(self, data): + if isinstance(data, tuple): + return tuple(self.value(d) for d in data) + return self.value(data) diff --git a/polaris/dataset/_dataset.py b/polaris/dataset/_dataset.py index a0ccd522..e22f1ba8 100644 --- a/polaris/dataset/_dataset.py +++ b/polaris/dataset/_dataset.py @@ -12,6 +12,7 @@ from pydantic import ( Field, computed_field, + field_serializer, field_validator, model_validator, ) @@ -70,7 +71,7 @@ class Dataset(BaseArtifactModel): # Public attributes # Data table: Union[pd.DataFrame, str] - default_adapters: Optional[List[Adapter]] = None + default_adapters: Dict[str, Adapter] = Field(default_factory=dict) md5sum: Optional[str] = None # Additional meta-data @@ -90,11 +91,24 @@ class Dataset(BaseArtifactModel): @field_validator("table") def _validate_table(cls, v): - """If the table is not a dataframe yet, assume it's a path and try load it.""" + """ + If the table is not a dataframe yet, assume it's a path and try load it. + We also make sure that the pandas index is contiguous and starts at 0, and + that all columns are named and unique. + """ + # Load from path if not a dataframe if not isinstance(v, pd.DataFrame): if not fs.is_file(v) or fs.get_extension(v) not in _SUPPORTED_TABLE_EXTENSIONS: raise InvalidDatasetError(f"{v} is not a valid DataFrame or .parquet path.") v = pd.read_parquet(v) + # Check if there are any duplicate columns + if any(v.columns.duplicated()): + raise InvalidDatasetError("The table contains duplicate columns") + # Check if there are any unnamed columns + if not all(isinstance(c, str) for c in v.columns): + raise InvalidDatasetError("The table contains unnamed columns") + # Make sure the index is contiguous and starts at 0 + v = v.reset_index(drop=True) return v @model_validator(mode="after") @@ -110,9 +124,7 @@ def _validate_model(cls, m: "Dataset"): raise InvalidDatasetError("There are annotations for columns that do not exist") # Verify that all adapters are for columns that exist - if m.default_adapters is not None and any( - adapter.column not in m.table.columns for adapter in m.default_adapters - ): + if any(k not in m.table.columns for k in m.default_adapters.keys()): raise InvalidDatasetError("There are default adapters for columns that do not exist") # Set a default for missing annotations and convert strings to Modality @@ -141,6 +153,16 @@ def _validate_model(cls, m: "Dataset"): return m + @field_validator("default_adapters") + def _validate_adapters(cls, value): + """Serializes the adapters""" + return {k: Adapter[v] if isinstance(v, str) else v for k, v in value.items()} + + @field_serializer("default_adapters") + def _serialize_adapters(self, value: List[Adapter]): + """Serializes the adapters""" + return {k: v.name for k, v in value.items()} + @staticmethod def _compute_checksum(table): """Computes a hash of the dataset. @@ -160,7 +182,7 @@ def _compute_checksum(table): df = df[sorted(df.columns.tolist())] # Use the sum of the row-wise hashes s.t. the hash is insensitive to the row-ordering - table_hash = pd.util.hash_pandas_object(df).sum() + table_hash = pd.util.hash_pandas_object(df, index=False).sum() hash_fn.update(table_hash) checksum = hash_fn.hexdigest() @@ -188,7 +210,7 @@ def columns(self) -> list: """Return all columns for the dataset""" return self.table.columns.tolist() - def get_data(self, row: Union[str, int], col: str) -> np.ndarray: + def get_data(self, row: int, col: str, adapters: Optional[List[Adapter]] = None) -> np.ndarray: """Since the dataset might contain pointers to external files, data retrieval is more complicated than just indexing the `table` attribute. This method provides an end-point for seamlessly accessing the underlying data. @@ -196,18 +218,28 @@ def get_data(self, row: Union[str, int], col: str) -> np.ndarray: Args: row: The row index in the `Dataset.table` attribute col: The column index in the `Dataset.table` attribute + adapters: The adapters to apply to the data before returning it. + If None, will use the default adapters specified for the dataset. Returns: A numpy array with the data at the specified indices. If the column is a pointer column, the content of the referenced file is loaded to memory. """ + adapters = adapters or self.default_adapters + def _load(p: str, index: Union[int, slice]) -> np.ndarray: """Tiny helper function to reduce code repetition.""" arr = zarr.open(p, mode="r") arr = arr[index] + if isinstance(index, slice): arr = tuple(arr) + + adapter = adapters.get(col) + if adapter is not None: + arr = adapter(arr) + return arr value = self.table.loc[row, col] @@ -409,17 +441,17 @@ def __getitem__(self, item): if isinstance(ret, pd.Series): # Load the data from the pointer columns - if len(ret) == self.n_columns: - # Returning a row - ret = ret.to_dict() - for k in ret.keys(): - ret[k] = self.get_data(item, k) - - if len(ret) == self.n_rows: - # Returning a column + if ret.name in self.table.columns: + # Returning a column, the indices are rows if self.annotations[ret.name].is_pointer: - ret = [self.get_data(item, ret.name) for item in ret.index] - return np.array(ret) + ret = np.array([self.get_data(k, ret.name) for k in ret.index]) + + elif len(ret) == self.n_rows: + # Returning a row, the indices are columns + ret = { + k: self.get_data(k, ret.name) if self.annotations[ret.name].is_pointer else ret[k] + for k in ret.index + } # Returning a dataframe if isinstance(ret, pd.DataFrame): diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py index ae90e14b..488ecbec 100644 --- a/polaris/dataset/_factory.py +++ b/polaris/dataset/_factory.py @@ -7,10 +7,11 @@ from loguru import logger from polaris.dataset import ColumnAnnotation, Dataset +from polaris.dataset._adapters import Adapter from polaris.dataset.converters import Converter, SDFConverter, ZarrConverter -def get_dataset_from_file(path: str, zarr_root_path: Optional[str] = None) -> Dataset: +def create_dataset_from_file(path: str, zarr_root_path: Optional[str] = None) -> Dataset: """ This function is a convenience function to create a dataset from a file. @@ -39,11 +40,11 @@ class DatasetFactory: Tip: Try quickly converting one of your datasets The `DatasetFactory` is designed to give you full control. If your dataset is saved in a single file and you don't need anything fancy, you can try use - [`get_dataset_from_file`][polaris.dataset.get_dataset_from_file] instead. + [`create_dataset_from_file`][polaris.dataset.create_dataset_from_file] instead. ```py - from polaris.dataset import get_dataset_from_file - dataset = get_dataset_from_file("path/to/my_dataset.sdf") + from polaris.dataset import create_dataset_from_file + dataset = create_dataset_from_file("path/to/my_dataset.sdf") ``` Question: How to make adding meta-data easier? @@ -60,23 +61,26 @@ def __init__(self, zarr_root_path: Optional[str] = None) -> None: zarr_root_path: The root path of the zarr hierarchy. If you want to use pointer columns, this arguments needs to be passed. """ - self._zarr_root_path = os.path.abspath(zarr_root_path).rstrip("/") - self._zarr_root = None - self._table: pd.DataFrame = pd.DataFrame() - self._annotations: Dict[str, ColumnAnnotation] = {} - self._converters = {} + self.reset(zarr_root_path=zarr_root_path) @property - def zarr_root(self) -> zarr.Group: + def zarr_root_path(self) -> zarr.Group: """ The root of the zarr archive for the Dataset that is being built. All data for a single dataset is expected to be stored in the same Zarr archive. """ if self._zarr_root_path is None: raise ValueError("You need to pass `zarr_root_path` to the factory to use pointer columns") + return self._zarr_root_path + @property + def zarr_root(self) -> zarr.Group: + """ + The root of the zarr archive for the Dataset that is being built. + All data for a single dataset is expected to be stored in the same Zarr archive. + """ if self._zarr_root is None: - self._zarr_root = zarr.open(self._zarr_root_path, "w") + self._zarr_root = zarr.open(self.zarr_root_path, "w") if not isinstance(self._zarr_root, zarr.Group): raise ValueError("The root of the zarr hierarchy should be a group") return self._zarr_root @@ -95,7 +99,12 @@ def register_converter(self, ext: str, converter: Converter): logger.info(f"You are overwriting the converter for the {ext} extension.") self._converters[ext] = converter - def add_column(self, column: pd.Series, annotation: Optional[ColumnAnnotation] = None): + def add_column( + self, + column: pd.Series, + annotation: Optional[ColumnAnnotation] = None, + adapters: Optional[Adapter] = None, + ): """ Add a single column to the DataFrame @@ -130,10 +139,14 @@ def add_column(self, column: pd.Series, annotation: Optional[ColumnAnnotation] = annotation = ColumnAnnotation() self._annotations[column.name] = annotation + if adapters is not None: + self._adapters[column.name] = adapters + def add_columns( self, df: pd.DataFrame, annotations: Optional[Dict[str, ColumnAnnotation]] = None, + adapters: Optional[Dict[str, Adapter]] = None, merge_on: Optional[str] = None, ): """ @@ -159,12 +172,17 @@ def add_columns( annotations = {} annotations = {**self._annotations, **annotations} + if adapters is None: + adapters = {} + adapters = {**self._adapters, **adapters} + if merge_on is not None: self.reset() for name, series in df.items(): annotation = annotations.get(name) - self.add_column(series, annotation) + adapter = adapters.get(name) + self.add_column(series, annotation, adapter) def add_from_file(self, path: str): """ @@ -179,12 +197,16 @@ def add_from_file(self, path: str): if converter is None: raise ValueError(f"No converter found for extension {ext}") - table, annotations = converter.convert(path, self) - self.add_columns(table, annotations) + table, annotations, adapters = converter.convert(path, self) + self.add_columns(table, annotations, adapters) def build(self) -> Dataset: """Returns a Dataset based on the current state of the factory.""" - return Dataset(table=self._table, annotations=self._annotations) + return Dataset( + table=self._table, + annotations=self._annotations, + default_adapters=self._adapters, + ) def reset(self, zarr_root_path: Optional[str] = None): """ @@ -195,6 +217,12 @@ def reset(self, zarr_root_path: Optional[str] = None): zarr_root_path: The root path of the zarr hierarchy. If you want to use pointer columns for your next dataset, this arguments needs to be passed. """ + + if zarr_root_path is not None: + zarr_root_path = os.path.abspath(zarr_root_path).rstrip("/") + + self._zarr_root = None self._zarr_root_path = zarr_root_path self._table = pd.DataFrame() self._annotations = {} + self._adapters = {} diff --git a/polaris/dataset/_subset.py b/polaris/dataset/_subset.py index ee70089d..95f8c0bd 100644 --- a/polaris/dataset/_subset.py +++ b/polaris/dataset/_subset.py @@ -2,7 +2,8 @@ import numpy as np -from polaris.dataset import Adapter, Dataset +from polaris.dataset import Dataset +from polaris.dataset._adapters import Adapter from polaris.utils.errors import TestAccessError from polaris.utils.types import DatapointType @@ -73,7 +74,7 @@ def __init__( self.target_cols = target_cols if isinstance(target_cols, list) else [target_cols] self.input_cols = input_cols if isinstance(input_cols, list) else [input_cols] - self._adapters = self.dataset.default_adapters if adapters is None else adapters + self._adapters = adapters self._featurization_fn = featurization_fn # For the iterator implementation @@ -127,12 +128,7 @@ def _get_single( """ # Load the data-point # Also handles loading data stored in external files for pointer columns - ret = {col: self.dataset.get_data(row, col) for col in cols} - - # Format - if self._adapters is not None: - for adapter in self._adapters: - ret = adapter(ret) + ret = {col: self.dataset.get_data(row, col, adapters=self._adapters) for col in cols} if len(ret) == 1: ret = ret[cols[0]] diff --git a/polaris/dataset/converters/_base.py b/polaris/dataset/converters/_base.py index b51ceec5..8e3c64af 100644 --- a/polaris/dataset/converters/_base.py +++ b/polaris/dataset/converters/_base.py @@ -4,9 +4,10 @@ import pandas as pd from polaris.dataset import ColumnAnnotation +from polaris.dataset._adapters import Adapter from polaris.dataset._dataset import _INDEX_SEP -FactoryProduct: TypeAlias = Tuple[pd.DataFrame, Dict[str, ColumnAnnotation]] +FactoryProduct: TypeAlias = Tuple[pd.DataFrame, Dict[str, ColumnAnnotation], Dict[str, Adapter]] class Converter(abc.ABC): diff --git a/polaris/dataset/converters/_sdf.py b/polaris/dataset/converters/_sdf.py index f7339147..76eab5cc 100644 --- a/polaris/dataset/converters/_sdf.py +++ b/polaris/dataset/converters/_sdf.py @@ -6,6 +6,7 @@ from rdkit import Chem from polaris.dataset import ColumnAnnotation, Modality +from polaris.dataset._adapters import Adapter from polaris.dataset.converters._base import Converter, FactoryProduct if TYPE_CHECKING: @@ -122,7 +123,7 @@ def _get_name(mol: dm.Mol): # Get the pointer path pointer_idx = f"{start}:{end}" if start != end else f"{start}" - pointer = self.get_pointer(factory._zarr_root_path, self.mol_column, pointer_idx) + pointer = self.get_pointer(factory.zarr_root_path, self.mol_column, pointer_idx) # Get the single unique value per column for the group and append unique_values = [group[col].unique()[0] for col in df.columns] @@ -131,7 +132,7 @@ def _get_name(mol: dm.Mol): df = grouped else: - pointers = [self.get_pointer(factory._zarr_root_path, self.mol_column, i) for i in range(len(df))] + pointers = [self.get_pointer(factory.zarr_root_path, self.mol_column, i) for i in range(len(df))] df[self.mol_column] = pd.Series(pointers) # Set the annotations @@ -140,4 +141,4 @@ def _get_name(mol: dm.Mol): annotations[self.smiles_column] = ColumnAnnotation(modality=Modality.MOLECULE) # Return the dataframe and the annotations - return df, annotations + return df, annotations, {self.mol_column: Adapter.BYTES_TO_MOL} diff --git a/polaris/dataset/converters/_zarr.py b/polaris/dataset/converters/_zarr.py index def5d906..7d26ea61 100644 --- a/polaris/dataset/converters/_zarr.py +++ b/polaris/dataset/converters/_zarr.py @@ -39,7 +39,7 @@ def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: data = defaultdict(dict) for col, arr in src.arrays(): # Copy to the source zarr, so everything is in one place - dst = zarr.open_group("/".join([factory._zarr_root_path, col]), "w") + dst = zarr.open_group("/".join([factory.zarr_root_path, col]), "w") zarr.copy(arr, dst) for i in range(len(arr)): @@ -48,4 +48,4 @@ def convert(self, path: str, factory: "DatasetFactory") -> FactoryProduct: # Construct the dataset table = pd.DataFrame(data) - return table, {k: ColumnAnnotation(is_pointer=True) for k in table.columns} + return table, {k: ColumnAnnotation(is_pointer=True) for k in table.columns}, {} diff --git a/polaris/loader/load.py b/polaris/loader/load.py index 0472908d..2bec36dd 100644 --- a/polaris/loader/load.py +++ b/polaris/loader/load.py @@ -6,7 +6,7 @@ MultiTaskBenchmarkSpecification, SingleTaskBenchmarkSpecification, ) -from polaris.dataset import Dataset, get_dataset_from_file +from polaris.dataset import Dataset, create_dataset_from_file from polaris.hub.client import PolarisHubClient from polaris.utils import fs @@ -39,7 +39,7 @@ def load_dataset(path: str) -> Dataset: if extension == "json": return Dataset.from_json(path) - return get_dataset_from_file(path) + return create_dataset_from_file(path) def load_benchmark(path: str): diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 3ed5ee3f..5a05b0f5 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -4,7 +4,7 @@ import zarr from pydantic import ValidationError -from polaris.dataset import Dataset, get_dataset_from_file +from polaris.dataset import Dataset, create_dataset_from_file from polaris.loader import load_dataset from polaris.utils import fs from polaris.utils.errors import PolarisChecksumError @@ -106,7 +106,7 @@ def _check_for_failure(_kwargs): def test_dataset_from_zarr(test_zarr_archive_single_array, tmpdir): """Test whether loading works when the zarr archive contains a single array or multiple arrays.""" archive = test_zarr_archive_single_array - dataset = get_dataset_from_file(archive, tmpdir.join("data")) + dataset = create_dataset_from_file(archive, tmpdir.join("data")) assert len(dataset.table) == 100 for i in range(100): @@ -137,7 +137,7 @@ def test_dataset_from_zarr_to_json_and_back(test_zarr_archive_single_array, tmpd zarr_dir = tmpdir.join("zarr") archive = test_zarr_archive_single_array - dataset = get_dataset_from_file(archive, zarr_dir) + dataset = create_dataset_from_file(archive, zarr_dir) path = dataset.to_json(json_dir) new_dataset = Dataset.from_json(path) @@ -151,8 +151,8 @@ def test_dataset_caching(test_zarr_archive_single_array, tmpdir): """Test whether the dataset remains the same after caching.""" archive = test_zarr_archive_single_array - original_dataset = get_dataset_from_file(archive, tmpdir.join("original1")) - cached_dataset = get_dataset_from_file(archive, tmpdir.join("original2")) + original_dataset = create_dataset_from_file(archive, tmpdir.join("original1")) + cached_dataset = create_dataset_from_file(archive, tmpdir.join("original2")) assert original_dataset == cached_dataset cache_dir = cached_dataset.cache(tmpdir.join("cached").strpath) diff --git a/tests/test_integration.py b/tests/test_integration.py index b2f4626c..7b96a4e3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -50,7 +50,6 @@ def test_multi_task_benchmark_loop(test_multi_task_benchmark): x_test = np.array([dm.to_fp(dm.to_mol(smi)) for smi in test.inputs]) y_pred = {} - print(multi_y) for k, y in multi_y.items(): model = RandomForestRegressor() From 7b0263d92dec57a3b23ae8b22605eb05516e1dc6 Mon Sep 17 00:00:00 2001 From: cwognum Date: Thu, 14 Mar 2024 15:33:53 -0400 Subject: [PATCH 6/9] Clean run of the tutorial notebook --- docs/tutorials/dataset_factory.ipynb | 122 ++++++++------------------- polaris/dataset/_factory.py | 1 + 2 files changed, 36 insertions(+), 87 deletions(-) diff --git a/docs/tutorials/dataset_factory.ipynb b/docs/tutorials/dataset_factory.ipynb index 28220c2f..dcf2bd35 100644 --- a/docs/tutorials/dataset_factory.ipynb +++ b/docs/tutorials/dataset_factory.ipynb @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "278ce19e-0b47-43f1-9876-b3b69a2154e1", "metadata": {}, "outputs": [], @@ -60,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "d8b3087e-8c50-45b4-ada7-44bf783cc929", "metadata": {}, "outputs": [], @@ -70,10 +70,18 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 4, "id": "0776f067-d01b-4b7c-89f6-a3c817f934fb", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to find the pandas get_adjustment() function to patch\n", + "Failed to patch pandas - PandasTools will have limited functionality\n" + ] + }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZJUlEQVR4nO3daVRUV7428KcoJsEogzKoqEGNA6IROlmixAEFkTAognMwU5t4jencmNtqnAhxwE7U+LbRqFfTGLEJgwSnqKgdlwja7dAK6DXIKCJDREQJFFXUeT8U7YCaCHUGlef3SXdV7f2vtWo97HPOPvuoBEEAERG1lInSBRARPdsYo0RERmGMEhEZhTFKRGQUxigRkVEYo0RERjFVugAi2Wm1OHQIly5BENC3L/z8YG6udE30DFNx3Si1Ljk5CAxERQX+8AcAOH0a9vbYvRt9+ypdGT2rGKPUmuh0GDgQtrbYvx/t2gFAdTUCAlBejuxsmJkpXR89k3hulFqTAwdw8SLWrm3MUADt2mHdOuTkYO9eRSujZxhjlFqTjAy0b49XXnmg0dMT9vbIyFCoJnrmMUapNamshLPzI9o7d8aNG7JXQ88Jxii1JlZWqKx8RPuNG2jbVvZq6DnBGKXWxN0dFRVNJ55lZbh+He7uCtVEzzzGKLUmISFo2xbR0Q80rlgBa2uEhipUEz3zuPyeWhNbW2zZgunTUVgIX1+oVEhNxa5d2L4ddnZKF0fPKq4bpdbh+HFoNBg9GgDOnMHXXyMrC4KA/v0xe3bjUnyiFmGMUisgCPD0xLlziIvDpElKV0PPG54bpVYgORnnzsHZGcHBAKDVIi1N6Zro+cEYpeedICAqCgAWLUKbNgAQE4PXXsOsWcrWRc8NxigpQxCE2tpaOUZKTMT583BxwTvvAIBWixUrAGDECDlGp1aAMUoK2LNnj6ur65IlSyQfSa/HsmUAsHAhLCwAYOtW5OfDzQ3h4ZKPTq0DY5QUYG9vX1BQ8P333+v1emlHio/HhQvo2hVvvQUA9fVYtQoAIiNhwh8/iYO/JFKAl5dX9+7dr169mp6eLuEwDQ2NZ0UXL27cmHnLFhQUoH9/LrYnETFGSQEqlWrixIkA4uLiJBwmLg6XLqF7d0REAIBG03j/UlQUp6IkIv6YSBmTJ08GEB8fr9PpJBmgoaHxrOjSpY1T0U2bUFyMQYMwbpwkI1JrxRglZQwaNKhv374VFRX/+Mc/JBkgNhb/93/o2RPTpwNAXR3+8hcAiIyESiXJiNRaMUZJMdId1+t0ujJDt4sXw9QUADZuxLVr8PBAUJDow1ErxxglxUyZMgXArl27NBqNuD3v2LHD6ccfvwwMxLRpAFBXhy+/BICoKE5FSXSMUVJM7969X375ZRsbh59+KhSxW61W+/nnnwPoNGUK1GoANf/7vygpwauv4vXXRRyIyIAxSkp6883UgoLLMTEvidhnTExMXl7eSy+9NGnSJAA1NTU9ly1b/uqrtYYrTkRiY4ySksaP76BSISUFNTXidKjValesWAEgKipKrVYDWL9+fWlZ2T61uo2vrzhjED2IMUpK6toVXl749VfRHm+8bdu2/Pz8fv36hYeHA6ipqVmzZg2AKMM6fCIJMEZJYYb9P0W5XK/VaqOjowFERkaamJgAWLduXXl5+dChQ0cbNmwmkgC3bSaFlZaiSxeYmqK0FDY2RnW1YcOG2bNnu7m5XbhwwcTE5M6dO66urhUVFUePHh05cqRI9RI1xdkoKczJCSNGQKPBDz8Y1Y9Go1m5ciWAqKgow1R07dq1FRUV3t7ezFCSFGOUlCfKcf3mzZuLi4vd3d3HjRsH4NatW1999RWAZbxATxJjjJLywsJgbo4jR1Be3sIe6urqVq1aBaBv374zZswoLS1du3ZtZWXlqFGjhg8fLmatRA/hA5ZJeba2mD4d1tZowS4ltbW1J06c+PLLL69duwYgPj7ezMysuLj4/PnzAOTYGZpaPV5iIuUVFyMjA6++im7dGlv0eiQlYcgQdO786I/k5t7Yty923759x44du3svaYcOHXx9fePi4tRqtU6n8/PzO3jwoCzfgFo1zkZJeadOYeJEvPoqMjIaNwLV6TBxIpKTH4jRhgZkZGDvXhw+jPLy6qtX/wTAxMTE09Nz9OjRgYGBQ4cOValURUVFJ06cALB48WJlvg+1MoxReiqYmCA/H1u34o9/bPpSaSn278f+/UhNRXV1Y6Ot7Yt//ONHr73mMXbs2A4dOtz//oiIiBMnTri4uHh7e8tSO7V2PKgn5SUlYfJkrFmDyEhcugQHB9TXw8ICycnYuROJibj7I3V3R0AAAgIwZEjjBngPu3nzprOzs06nKy4udnJyku1bUKvFK/X0tJg1C05O+OSTBxo7d4alJUaPxldfoaAAFy4gOhrDhj02QwHY2tqOHTu2oaHh+++/l7pmIjBG6elhaoqNG7FjB44evde4aBFu3EBqKv70p3sXoH7XtGnTAMTGxkpQJlFTjFF6igwbhilT8MEH0GobW+zt0aZNs/sJDAxs37795culP//8i7gVEj2MMUpPl9WrUVKCTZuM6sTS0vL998/X1xfFxnb4/XcTGYcxSk8XJydERcH4Gzh9fbvV1SE2FryGSlJjjNJTZ/ZsdO9ubCcjR6JzZ+Tm4l//EqEkot/AGCXlOThg1Kh7/1WrsXEjRo+Gg0PL+zQxadzxhNeZSGpcN0rK++tfERYGZ2eRuz17Fp6ecHDAtWu/tUCKyEicjZLCfvoJH34IT0/U14vcs4cH3NxQXo4jR8TpsLS0NDU1VZy+6DnCGCWFffopAHzwAczNxe988mTA6OP6/Pz8devW+fr6uri4hIaG1tXViVIbPTd4qENKSk5GRgYcHDBnjiT9T52K5cuhUjX7g4IgnD59Ojk5OSUl5eLFi4bGNm3ajBo1qrKyslOnTiIXSs8yxigppqEBhj2Yli7FCy9IMoSTE3JyHtgmSqvFnTuwsXl0tmq1+Okn/PTTupiYLwwbmAKws7MLDAwcN27cmDFjrKysJCmUnmWMUVLM9u3IzsaLL+Ldd6Ua4r//G5s347vvMH16Y8uBAwgOxq+/PnBz1K+/4sgRJCRgzx5UVWH48IZr16517drV398/MDDQ39/fzMxMqhLp2ccYJWXU1+PzzwFg2TJJzoreZWmJuXPx+uuwtW36Unk5du/GDz/gyBHcPeE5YADGjJm+Zs0IDw8PCcui5whjlJSxfj3y8+Hu3ngVSDojRqCkBPPnP3CDaVUV/PyQng69HgDUarz2GkJCMH48XF0BOABGrFmlVoYxSgq4cwerVgFAdHTjdvfSUavx9dcYPhwzZmDIkMZGGxuUlMDcHN7eCAzExInir1ql1oMxSgr44guUl8PbGwEBcgzn7Y2wMMyahTNn7jUmJ8PVFW3bylEAPd+4bpTkVlGBtWsBIDpavkHXrEF+Pv7613stAwYwQ0kcjFGS24YNuzt2rA8KwtCh8g3auTMiIxEVhYoK+QalVoL31JOsCgoK+vTpIwjqc+fy+vVzlHq4997DtWvYuxcAdDp4eKCuDjk5TRc8ERmDs1GS1ZIlSzQazZQp4TJkaBOmptiwAVeuyDwsPf8YoySfrKys2NhYc3PzJUuWKFKAtzfeeEORkel5xhgl+Xz66ad6vX7WrFmurq4yDKfX4+23G4/o74qJgSDwiJ7ExBglmZw6dWrv3r1t27ZdsGCBPCMmJGDwYLzzjjyjUevFGCWZzJ8/XxCEuXPnOjrKcVa0oQGRkQAweLAMo1Grxiv1JIfY2Njp06d36NAhNze3Xbt2Moy4dSvefRe9eiE7G9xXhCTFu5hIQkVFRcnJyQkJCSdOnACwcOFCeTK0vh7LlwPAZ58xQ0lynI2S+C5fvrxr166kpKQz99196eDgUFRUZGFhIUMB69bho4/Qvz/On5f8nn0izkZJPJmZuw8dWvi3v2VlZRka2rVr5+/vf+zYsbKyslWrVsmToTU1WLkSAJYvZ4aSHPgrI6NlZyMyEv36YcCAl1JSsrKybG1t33jjjfj4+JKSEj8/v8rKyo4dO74h14rNdetQVoZXXkFQkDwDUmvH2Si1iF6PkyeRlIRdu1BQ0Njo6NjN0/PQ4sUjR440/c8TjauqqrRabX19vVqtlqGuqiqsXg0A0dEteQQTUQswRuk3VVdj505cuACdDn36YPJkdOqE/Hx4e6OkpPE9Xbpg/HhMmABv7zZqte99n75165adnZ2ZmdmtW7cuXbrUt29fqev94gtUVmLYMPj4SD0UUSNeYqLHy8zEmDGwtMSoUTA3R1oacnMRHw9/f3TpAnNzhIQgPBxDhjQ5B3nz5s09e/YkJCSkpqZqNBpD47JlyxYuXChpvRUV6NEDt28jPR1eXpIORXQPY5Qeo6EBbm5wcsLBgzBcGhIEzJyJhATk5KC+/oHnbRqUleGHH+oPHGi/f39dfT0AtVo9fPjw3r17b9y4cdCgQWfPnpW05KVLr0RF9QwORkqKpOMQPYAxSo9x+DB8fZvO66qq4OyMVavw4Yf3GouLsX8/9uzBgQPQ6QB88PLL/7a2Dg8PnzRpkpOTk0ajcXBwqK6uvnLlSo8ePSSqt6io6KWXXurff9i33+5zd+diUZIPz43SY5w5A1NTvPLKA402NnB1bXwWR34+EhOxaxdOnYLhj7GlJQICMGHC/wsKMrnvOZwWFhavv/763//+9+Tk5E8++USieqOiojQaTe/eHZmhJDMueKLHqK6GnR1MH/pD6+iIW7cAYPly/PnPOHkSlpYIDERMDMrKkJKCiAiTh55lHBoaCiApKUmiYnNycmJiYtRqtVJb8FFrxtkoPUb79rhxAzpd0yQtK0PXrgAwdSrq6jBhAvz9f3fjuYCAAGtr61OnTl29etXFxUX0YpcsWaLT6WbOnNm7d2/ROyf6bZyN0mN4eKChAdnZDzSWlyM3F56eAODjgx07MH78k2zeaWVl5efnJwhCigRXfzIzM+Pj4y0tLRctWiR650S/izFKjzFyJHr1wqefGq4aNVq6FBYWmDKlBf1NmDAB0hzX390NWop5LtHv4pV6erwzZ+DvDycnjB0Lc3McO4bTpxEXh5CQFnR2+/ZtBwcHrVZbUlLi4OAgVo3//Oc/Bw8ebG1tfeXKFXl2MiVqgrNRejxPT1y6hDfewLVryMmBjw8uXmxZhgJ44YUXfHx8GhoaxD2uX7BggSAIH330ETOUlMLZKMln27Zt77zzzpgxYw4cOCBKh4cPH/b19bWxscnLy7N9aHkAkTw4GyX5hISEmJqaHj16tLKyUpQOFy9eDGDevHnMUFIQY5TkY29vP3z4cK1Wu7fJ4zqb7+zZsxERESdPnnRycpozZ44o5RG1DGOUZGXk9frs7OzIyMjevXt7enp+99139vb2wcHB1tbWotZI1Dxcfk+yCg0NnTNnzsGDB6urq5/wuUyCIJw8eTIxMTEpKamwsNDQ2KlTJ1dX17S0tDZ85DwpjbNRkpWjo6OXl5dGo/nxxx9/+516vT4tLW3+/Pm9evUaMmTImjVrCgsLu3Tp8uGHH6amphYWFr7//vsAysvLZSmc6LE4GyW5hYaGpqWlJSUlTZo06eFX9Xp9enp6QkJCYmJiyX92hnZxcRk/fnx4ePjQoUNV/9nU3rDCqaysTLbKiR6JC55IblevXu3WrVubNm0qKiqsrKwMjQ0NDRkZGQkJCQkJCdevXzc0duvWLSQkpEl63pWZmTlgwAA3N7e7T9AjUgRnoyQ3FxcXT0/P06dPHzp0KCgoyJCe8fHxpaWlhjd07949ODj4kelZUVHRsWNHw78Nt0JxNkqK42yUFBAdHb1gwYKePXtWVlbeXUPap0+fsLCwsLCwgQMHNnl/UVFRcnJyQkLCyZMni4qKOnXqBKChocHCwkKv19fX15s+vKEfkVz44yMFDBs2zNnZOTc3VxCEfv36BQUFBQYGent7N3lbXl5eYmJiYmLi6dOnDX/vra2tMzMzDTGqVqvt7e3Ly8t/+eUXJycnBb4GEQDGKCkiIyPj+vXrHh4esbGxffr0afJqQUFBSkpKQkJCenq6IT3btGkzatSo8PDw0NDQtm3b3n2no6NjeXl5WVkZY5QUxBglBWzfvh3AkiVL7s/QvLw8w/NET5w4YWixsrLy8fEJDw+fMGHCI9fYOzo6ZmZm8vQoKYsxSnI7d+7chQsX7O3tx44da2hJSkpaunRp9n+2iLa1tQ0ODg4LC/P19bUwPJT0MdzdV5SVRVZW9pe8aKLHY4yS3GJiYgBMmzbN3Nzc0KJSqbKzs21tbQMDA8PDw8eMGXP3pd/zSmYmrl2TrFaiJ8AYJVnpdLq4uDgAM2bMuNvo7+9/8OBBHx+f5l5wN2wxymN6UhZjlGS1b9++srIyNzc3Dw+Pu42GJzW1oDfDJvq8HZSUxXvqSVaGI/o333xTlN44G6WnAWOU5FNZWbl//35TU9Np06aJ0iFjlJ4GjFGSz86dOzUajZ+fn7OzsygdGg7qGaOkLMYoycdwRH//xSUjOThApUJ5OXhLMymI99STTC5evOjm5ta+ffvr16+LuNeyrS2qqnDjBuzsxOqSqHl4pZ5ksnNnr6FD84cPzxB3v3pHR1RVoayMMUqK4WyU5KDXo1s3FBcjPR1eXmL2fOQITE3xhz+AD2QipXA2SnJITUVxMXr1wuDB4nRYV4eICLi5YenSe42ffYbOnfHuu+IMQfSEeImJ5BATAwAzZuChPexbSKdDQgIiI3H48L3GY8dw9qw4/RM9OcYoSa66GikpUKkwdarIPXt5YfZsaDQid0vULIxRktz33+PXX+HjgxdfFLnn//kfVFcjOlrkbomahTFKkrt7RC+6du2wciWio5GTI37nRE+IMUrSys9HejqsrTF+vCT9z5gBT0/8139J0jnRk2CMkrS+/RaCgPBw3Pfsj5a4devR7SoVvvkGx45h1y6j+idqMcYoSUgQEBsLGH1En52NXr2wadOjX+3fH7Nn489/Rn29UaMQtQxjlCR07Bjy8tCtG4YNa3knBQXw80NFBQ4deuy98599htpaZGS0fBSiFmOMkoRSUgAgIgImLf2h/fIL/P1RUoIRIxAb+9hlp+3aYfVq6PUtHIXIGLyLiSS0ejUCA9G7dws/fvs2/P1x+TIGDEByMiwtAWDvXiQmYsMGbNqE+5/NPHkytFrx11QR/S7eU09iCgqCRoPduxsjD8Bf/oKcHGzZ0uyu6usRFIRDh9CjB9LSYHgQfVoa/PxQW4u//U2SFVRELcCDehJTZiZSU7Fy5b2Wq1fx88/N7kevx/TpOHQInTohNbUxQ7OyEByM2lrMnMkMpacIY5REFhCA6GhcumRUJ0uX5iQkwMYGP/7YeJyenw8/P9y8iZAQfP21KJUSiYMxSiIbMwajRmHWrJbvSL9o0aLly/uMHv3PPXswYAAAVFRg7Fhcv46RIxEXh2Y+hplIWoxREt+aNcjIaFwx2lwbNmxYvny5iYlq1qxib28AqK5uvNA0cCB27bp31pXoKcEYJfH16YOPP8bcuaiqat4Hd+7cOWfOHJVKtXnz5tDQUAD19QgLw9mz6NkTBw/CxkaSgomMwRglSSxeDCsrREU14yOHDx9+66239Hr9F1988fbbbwNoaMC0aUhNbbzQZHicMtHThjFK4qisfOC/VlZYswbr19+7TP/ba+NPnTo1bty4+vr6+fPnz507F4AgCEuW7P/hB9ja4uBBdO8uTd1ERmOMkrFqazF/Pvr2RWnpA+3jx8PPD4cOAUB5Ofr1Q1zco3vIzs4OCAioqamJiIhYsWKFoXHhwoUrVrw+ePCqvXvRv7+k34DIKIxRMkpaGtzdsWoVqqqQnt701a+/hpUVAGzYgMuXMWUKZs5ETU3TtyUlJVVWVo4bN27btm0qlQrA+vXrV65caWZmtmCB+5AhMnwPIiMIRC1SWyvMmyeo1QIg9O8vnD0rCIJw/LhQVPTA286dE/71L0GvFzZtEqysBEB48UXhxImmvW3fvr22ttbw7x07dpiYmKhUqm+//Vb670FkLMYotcT588LAgQIgmJoK8+YJdXVP9KnMTGHAAAEQzMyE1asrdTrdw+/Zu3evmZkZgNWrV4tcNJE0GKPUPFqtEB0tmJsLgNCjh3D8ePM+XlcnzJsnmJo2DBrk4+XllZube/+rJ0+etLa2BrBw4UIxiyaSEmOUmiErS/D0FABBpRJmzhTu3GlhP0eP5jo7OwOwsbGJi4szNGZmZtrZ2QGYMWOGXq8XrWgiiTFG6YnodLqVK1cOGHAHEFxdhWPHjO2woqIiJCTEcII+PDw8KyvLxcUFQHBwsFarFaNkIplwozz6fT///PObb76ZkZHRq9d4X9+kVatURj5Y6a7Nmzd//PHHNTU1arW6oaFh5MiR+/fvt+T9nvRMYYzSbxEEYcuWLYakc3Jy2rJlS2BgoLhD5OXlTZ06NTs7297e/t///rcN7/ekZw1jlB6rsLDw7bffPnr0KIDw8PBvvvnGcO5SdHfu3CksLOzZs6eFhYUU/RNJijFKj7Z9+/YPPvjg9u3bDg4O33zzzXiJHjNP9Ozjxo3UVGlp6Xvvvbd7924AYWFhGzdu7NChg9JFET29GKP0gOPHj4eEhNy8edPOzm79+vVTpkxRuiKipx0P6ukBVVVV7u7ubm5uW7du7dy5s9LlED0DGKPU1NWrVw1LOInoSTBGiYiMwo3yiIiMwhglIjIKY5SIyCiMUSIiozBGiYiM8v8BBq3eN4zITYIAAB+5elRYdHJka2l0UEtMIHJka2l0IDIwMjMuMDkuNQAAeJyFmWdUFGvw5hlyjkOOZgREFCPIW4MoCJjjvYqKCiYUEBVFFBEMiGSQHBQQJaigZPCtVxGzYsSMCYmSRJDMztzzP7t79sv2lzrT3TPn13Weqnqqp4NerRPiHwpC/+eQ/5/P/hxxAzN+FBGVcDYURBFx5zmCyPm/TvwXRf/fKGnwXxT+3/f9zw/9/67LCHGEOMLOwiI7hUVEnUXFdgqLiTuLSxgKi0s6S0oJSUo7S8sYCkvLOsvKCYlxnGVFnbnC/C+Kcfi3i4mLS0rLioqpC40VEprOP8v572EqmySZt5gjE5eNxu7cCFSbqM+sMq/TqrmdNNncit17bwFlS6tI/EMOy65OgHsqkWQodg++9vlIso4a4TzrVJz0XRx32R/Be9NC6IIjlTgjvxVL++Rw6agqc/5WQFtW7AL9s0VommOGwg+7wfO1Fkv6xYXYfXFgKt5Ge9ZGQx9T4dG4elqquRM66Fly8XQnCV33B0otVaFxPwWOAFd+i7Q4c4txYGs3x+LCB8GYfVSPBZ6upEYSb6jNG8Lm8aZCpWgduTlfmLWqnwdVjzRib7ML6abXZFzFePzYkYxfT0ng9rSjGJZ7hhYFlmHSuHb88VkGTe247PHSh3SM1j7QXVqIU5Ks0fbpIPDKNVimixScKE6AwaVttIZ3DuouqPFy332ng6d3gs/EOPLStpU4Lu+BLbdU4fnmW8ARpF1+mbEsE+c5sPtTQ7BrWzyGR+iw0vQ0um+RLM7bMZfJ/OsAn7YnE3eFQVRYlQbBq3eRpq3bcNGbb6T6xgwMkU7EzdliGPLXF7VXn6VbHSmO6rbgugdyuKlRjZ14e4UWrN8OojklaHFZH/9Z0QWPWnWYlz8XDh2Igy7tXhrcFgfLXijzPOoaaOcVZ7juHkMa6nvImTNt8MNSB17uR+CICGjdzkiznyqOTHd5GLpujMAqKX322L2Iyi9/R2uDCRtSMgX7Yz9JrbUwm+IYDltDUsnhwx5YlInkis8UjDdJwXJpGdzQGYDNR4/Q/LwyHLe4Bz+86KFXB5TZ0vR7tC3PEy4q30SPibY4yXwAngSps9L3XHinFwfcyb/pho8RQC1VeVVlw1RawwvE4v3Io2cfyKPZw7A0RRFKn+YDR1RAy9snxQ46OrJn9yPxjVE4brumzwyiblKDAx/oP6GEVd2cDtu4n8mvRmE2/n00HA1JIrZG7li/6BHp6TPCMywVffwkMarhGHJ7j1N1tTL8496B3ieGqVugCpsleocG3/KCA92FWCvDw9SXQ/AwRZOlFMrDqyPxsIs7QG1MIuCCDZfn6dlOrb7tgZoZwcTN6gtZrtcPb7xVIF6iGDhiAtqGfAm2L3oJWz4xFC+YRuI2Zz02pfwWLYUmml41j8m/mwL705+SkyuFWNulKNgwJphoTtqFby/fI1LJZqhUmYQqnnK42CgArdqO05SIChxd3YlJWT3UYJ0qizz9kF7bsA9y8SbKN89Fn4kjkOGqzvTVZcDkWgK483X71SwUHI9yeQ4ddTR6yAVKesNI2clGYq79B2rWqMLyOeXAERfQ5ipJsMC/jmzt4SjcdyEM33fqshI1SmvVvtDoS/PYgY9mcP3SC+LuJMTutMSARFo0eTvNBbPya8isvMn451Qiv8qk8N7nE+h14xR1DalEZc9O3BYui+nSqsxg3xPq/sUdhrwLcV434NC8fhh212Dt+yUB3yRB2uku+qAsBDb1qfI2wUc62Xo7zIu9SNobm8lz79/Qv4kL5q78KpMQ0L5PlWYi2o5MaXEQpqnGYUqtFutYkEt/Zsqgd94cFtfCA+X8dLJ12wAmc5KhpmA/oWZbMKLuLSEHp2OCUhyWbpTGgi5fND7jT2e9RFya2Yp9raN0iYcme3Api+oYekDRrzI0Fx+HFq//wGmeHrNPUoOvGfFgcG6AeifGAK9FhdeYy0H/RS6Q2htHKnkdpDmsEyTvK4P+otvAkRTQnn0kwZJtF7Om3gjUsIvETI4um72vkrLk39Sj3oKtMpoDH7sryePsYSRj4mHZwSByWMMVF4W+IpvfmeAd2USMyJHDSfE+WKl4hFqYIu5QbkaXN3+o2HMNlm9yjRbU7wPyvRgTOWa46nQvaBjrsKIyJVC34Xewyx20iK+wS6+4vNrFX6nwMlcIYOdJf3UXOQhdMKlIA1Ja+T1BSkC7+rEE65y8hAU9jMR9Y6JwMFOPSYsW01H3EdoSPY9ViVrCzjf5xGm6EOMqJ4HkBn8SxXbissbPpG7nFFTUTsF+fUmUVT+E3pfP0KrVlbg/+ydOduunZrfUWUJECfV/7gH5q0rRfuEU1PPqgz/rdJh3nywM98eDXckv+vpxGHzo5fLc279R14mbYW1vLLH6t5e8kGwDekUTxu25AxxpAa3dJSl27LADe/MsHC8URqKSiAEbO3yBLs/opHLtVmyZsjVEWt0lY7kc5jQ3EQauBJHWDXtwKP8tseyYihvOpaGPiBgGLfTFupbj9JlvOR56+gsfL5bBehkuuxxfTEUeeYJvTBFum2yK3S4DoF+pyaxdFaD2zXn4PaGDXpoTDhdGVXhPhXpoyQ5XmBR+hrzObieveN3Qt1UbPi/kV5mMgNZhujQT+2jPfm+IRPFXUdiwQp/Vz0qlwcnD1NDBit3InA9vE8rI6tdC7GRJCizuDCBv7u3Ggt315K+IGdq6pOK7j0K4qNQHZ5w+QuumlqECtOK41bIodEuF5QwV0gp7L0icW4hNUWb4TrMPHpprsYDdymC6NwGk2r/StKxQUFVU5fmf7aR9G7bA/JUhROxDP9k+0gFm9voQ8JpfZbICWj8xeda+yoE9uxWMhSHxWLBXj62Oj6XRfrI4vsGSda9whHsHr5AJR4VYy9NUsFc8Qy7dc0Mvmy/kR64ZdtqmoVIRB3WKvDAlI4RG11SgdG4TTlqjgLErVBl3wXkK/Dr3V+LnvJCL9wN+g7KMHlN6MA4uOsTCqZMc/Ho3GZ6Kq/BORcrgh/5dcKg7kJTe/EpiZvXA08/q4EdKgCMnoLVOl2LVDxxY8/ho3MWLxgkf9Nimvks03kQYbUbnsfNpNjBG9CrR8hNiXzJTodTQj0jp7MJm42bSxZmKdS4pGBoujBMCvdHzqw9doF+BcVua8OU6WWxMUmUr716l22S84G1/IW4MMsRhTh9MeKnF4iqUQeddAthBHfUVDQOTAVVeukU7zb63Ca5tjCCir0ZJ5PtfYDYyBu4jA47ATMoHKMqxnB327HJUEOq4xuH8NdrMjl2lpWVieG+qJfu6eCak9VWQ9b1CbIliLOT0RJPmjF34NOcduSM/HX/+jMNjSRJYoeiP3Lbt1Ph5GfpaduGqv+10zCcVtjvwKp0RfgDqFhXitVtT8YDuABhc12QmQTqgkhEH43Z00KeesSD9UIVX6CKE/cke8PboaXJpoJFUlA7Am89KYDvAn2UC2yv/slyMtRU7sMjWc6gYw59npTps4rNL9M+LZlqtYsneH+VBcmA1mTRmFDedToa2zGAypdgVZ0V9ILeyTZBXk4AZ02RQ84AfXis4TM83MJSvakV68je1fKTJ/lEqpbVaHvBTqBRx7RSUqOyDVx7azPaILBSoJkOjqRD2CUVDrJkqz/hYHdUo3whOJ9NISdtHsia0FfJmKcBf2UrgKApopR5Ks0Vt9mxPVyi+t4vCLl891mJ2kWad6qTWslbMR5MHZ548ILWMw+YkJEJ1azTZyXNDCas60hdpgrOXpGDHBA4WXfbBS17+tMe6BO89akXxP0IYk6/McmNLqFs/P7fHizHz+0y8d78fjo1qsev7VCBvcRz4v+qkpD0SjhxQ4ZVP7aZSzrtgs3o06XH6SebV9ILFKVWYcIDfE5QEtOvkpNgR/UUsYn4I/usRiQsjdJlkYhTdPomD2s8s2cSZS+C3bwEJDh/B9kOpIHIhlLgscMG9478SNQ1jPCKUhGOzxFB/hQ/mfYqkd2wYXoptQeF9CjgcrcG4m7NpWLQr5P3hu3R9Q3Qtbofx4QZM/IE67N8cC2lSorjSPxY+NSjxFlwVQbXDO2Dbm1hSLfadZN3shCAnGRivw3fjygLafDsJluqyiC0vC0FtzzAM2KjLUt5n0COjzfT6TSt2vG8e8CbVkGxtEZa/PQmkTYNIhOl+/CD3hrxZaIq3ryein540lo31xTzuFtpoLqiyXyi7v56qNqoyOc8i+s3zIPg0FKLfR3M0XjIIUwo02eO9qiDtmgSTT7TTWIiCK99UeUL32+nTuo0AswOJYVQt2bmvA5btkYLp9XzdqghojXZLsbO/7JnJxSh8WxmFlY902YXym7Suc4B+JPPYi1dzQCWxkkw/JMQ+xCbAkbyzZHH2LvSp+kgiXkzBTauT8B8PSTT4fgiHQ/3oVF45jkS04KsOCdTu5bIF16/Tz2aeEL2gEBVGpuITkb+gsEOLvfNWBoXhOEh+0UQfKkXB8imqvPs2DXSPnCvUTgolxhe7iUV9F0R368PzKxXA4Qpoi86KsUYXe5ZeHYUqxqE47KDHsp3y6NqLzdT9rRWbEW8FbBEjtjs4LCQ4Ef41DSa7ZXejx+PPZLTFGJ2y+ZvOUSmsNjiOcdxQ+tiO4rm3rfg6qY+mH9FkoebV9Jr6TsikxWh20xyLHf/C141arK1CAkyWxMGcZZ3U6u5ZMBrL5V23/kV7zTeDU0EUKTjWSSx626BqnzqseMH3CaoCWkVPcTbnlQPbujoSsy6FY7WxAdMxiafrrg5TOS0rduusAwS+ukqmyXCY/6QMgKoT5NpJN7zY1k3EYo0wcHManpwihI/93fHz12A69wffa7V/x3tHuGhwTZVpt+fRq4EeUDKlDBP+mYQe23sgpkeXrWuRhiGjONhZ0Unp0Qj4oMTlBel9pCu2OcHhujgSdHaI7KtuBr8n+jA1oAo4agJaMlucBXOWsJUbonE3hGLvE32mr3eL6t35Tq9tJmz47hToKn9OAseLsLis88CVSyZC9W743qaWjD42ROn8VJT3l8FN7YfQJ9afqk6oQPn2VvwyKItzDFTZDPFbtKHCC8qHirAzdyaeXjQIbzma7Gu9PGTlJsCC8e3UelwoWIuo8eLutNDqf13hpVw4+XriJ/kypgvGeqlCtkMZcNQFtCfaJZmxxGL2+FoE6gdF4UCiLnPJqqRb3YfoqgPzmJryNAj4eouo8Cfv3SmxMOFmCGm5uBsdsmpJ0TpT9DBMwnu2shgj5IvprYfo2oxKDGttQc0NbTTPXp1ZOuTRHEsv6Nl7E88dMkc/flcq36rF9jI1kJFJhIcJP+kabjRInFLj7elrpAfnbwf3smCS3N5BNvb8BnMvTXj8ia9bDQGt41xJpnbanl15GoLXgiKx8aYum7kxhx5U76V7cyzZ0r/8KlOqInciR1Gbv5tuGRtGPozbjvuNXhH9TWbo/plP+0sac1b54iP0oRb1Faj2pRUHjw5RmxJVtsGmlOZOPwCLykvxbvE0tEwfAKNv2oyzTQEuqcSD9c4hKtYfA+XzVXhOP75S9VMuED0hnGwO+UYqnnZBepwCvBbm91tNAW0mV5rpJjuwFf+EIe8Lv4ON6DMp5VyakfeS2gUAez84DVZnNJFyb1EWkRQFR6uyiMVDL1SNfU4szhijcXYqyqwUw2P/HsP2XbuoBN9rtRt2o4UxBwelFdkTmfu0YtAHHPEGBoYuwlnjR+CFgzpbeUERwufFQ9aTLrrzYDicu6nKu6jYS7O63WFWTiDRHXxP0rcPwvtkOdB6UwAcLQFtXoMky9xqx2T8wvGARjhK1uizhVtD6Np/e2jSYmBtS+1AZsEdUtUuwrpZGuyvOUeSph/GiWe6iGa8ES7NSUX1/X00zPwomgRF0UK7EvQZ+YU3eiRx7hgu+3fFHVr2yQ2SJUvwy5tpeJDXB64OWuzuB1noE4+F2Om/ac63MDAnKrzLtl20b+ZG+F5yiATsbSIDX9og4IIazNbjezBtAa1WhjjTXOPIHsuGYZxQBI5O0WV32wupwo8e+uzDPBa+dxYcv3aLDNZw2PY5CVBjFUpS/XbjROvP5JivIdYJJeCd3RIofuogGnLdqdvsckzMbMSrOt10+nlV5tiUR/0O+kBQXSFeeT4TRwb7QMlfm5mVKsDKdYkQHtBOr2+NhFV5XJ6N0Ud6d9lmGG8aSpa/byQuju2wwFIRPOv5PkFHQCuySZxFfF7E/haFoZB+GKaf1WOrHqfQlJSfVKHFkv20WgiLKx4QczaCnsPJUOscQRba7sAdn7+TGL3JaNWchMtyRHDenMMYs+sE1WYV+FK5FfMTVXCgicuuzqik3zZ5QMZQKTp/mobHuH0guVyHkYVSECwbB6NxQ3TGsRAwXMbfImN76cf8baDVmUzcA1tIm3Q7TL+kBDK+fDeuK6CtWS7BTE45sqKgCJSSj0CvWzpM17KSwoIeOu6FJZs+aToEhlKS0sHf0PX4/XFONDnz2hVzfN+R03ON8KpfAsYoyeLpvoMoX7CXrllTibNfNOOy+8L43U2Nif24ShOOHwDVV0V4rs4URV37wbxEiynVKUFHcgJw7Trp2mnRMNVZlRdx+gvdtmwnLEmJI/eKO8je6t9Qm64BEbV8Wj0BbdywJIuycGQ/jobiIYNIDJfSZzUkk94o7KKFawmbeQagLrKaiL8UZqFFSfDSN4I8affAsAs/yNBjE5TvTMFrs0TwmcUBrLE4QKMjirH8Nt/VLONg3msVJh98gxoGHwS7zkLc02iOPVcGwLNKiykbK8GLcwng96WLzjGKAINGLi9X5jeVi90C6U5BZFtQPcl+1QFzLPkb+qZS4OgLaNfHS7Azt+3Y8W+hKP8oDEuu67KpLecp16efmmZYsakvbEFzVyl5tInDku1SIOthKOl8y5+8pQ3kSdlktJiehNGZwmgtdhjD3P3oinOVeHF8C/LmyqGmrSrb4VFCpTd4Qf6OCjxTbIrT9/bBi1+67MwsWYiNiYUCDQ5OroiAEUMVnp1WHy085QwzWiJI3LgfRNSgHV7+kgO17XzdGghof2yWYdv67dmE0mDksRj8wN8djhYVUzcpIex2m8f+0TKHaeZ3iPNlYba25DxsbIkjJ/k7b8fpWhK9ywTDDsZjcbs40nfHkWO3ne4NqsA2hS6cW9tLJ7tzmdadcqo0sg8CtUvwm+xsfBrWD1sfazGNu8owLT4elC2EMW5NLDxtVea1TvtNyY7dcDE+mPS1viWbu3rBUUMaGlYWAWeMgDbZUJwZqy5isy+GY9yrUDzD1WWetVdoeuUvusXQit0VmgdXd1aTJ0IiTGxFMnTeiST6np5493EzcYswQgOvJJwYKoY5LUfx8usT1ONcBf4Q/4UJkb+po6E6U75QSh8J7QejpkLckjAXvaf+hdn1fCU0yECSUyzov2qhlp9DYfIyZZ7cvM/03oktYHwziJhYdpLZ67ug6qEqtK7k+9uxAtrRZHH2R9We/W05i/6PgrH9jD5rfJpMxbhPaL+DNau8Mhfc5FpJ6koxZrLhPLTZJpMXH3wwvOEFmWVngpJKKSg9SwJzegLwQak7nb2sFFPru7HiUjfVvK3EfIc+ULkAbyg3KcY/rTbY0M/hzeSqMxc9MWhbEQtGBSN0x6wweOOvwltU2UD3znCBC6u9ya+CGlJd+AeO7ReBS3vzgTNOQLvophAL0F/C3v0bi0knTuJotgEba15ML7kW0+NiPAZ6c6B+cQMJUhdm1w7FwxPJUDKJ64EHf74g9+8YoW1xKgqJK2PX2iPIDh+g5zMq8GN4M05X7qYax9TZEdF3dPmhozBNtATDr9ih05AQL3m7Jtsq1ktGpiRDYFUrdfA7CeOmqfECR5BeCVsNB9zPE3HvH8S2uwHMqAysvc7P7fj/lJAgwXpXOrKCxBgsuh2G51z12dNxN+kdj2Zact6KYcMcWHHoHlnYy2EJ0xJgxbR4ss90N+4arCM7qg2xxCsVE3sl8N72AxiqeIRWD1fgzhVNeLREFs9xVZlaRj4tWLMfEtRKcHKEKfqE98PyHi1mVMuFt88SwffzL9p5ORzqX6rxhsJ66OB0V9g/J4o0FTUT2wudUDaiCd9E+a5mgoB2728x9mSnA7u5OAJD/4RitJsu63+dTQvke+mRsfOYUZYltEgUERfhUazjz7Kc96dJzI4dOND1mVi9NEL1XYn4aJEUztfyxW0vj1NOOmKJZTMWnpBF3w41dtK2lM6c4QHdE0rxVpExLlk0AE4vtdnMMmlwsUyEA1F99KRTGNhRLi/gdgNNX+QEBjppZJPHD7I7vAVgphz8tGbAmSig1ZCQYk+zHFnT+CgsLotAhWB9Vse5QnO0/9JHFoRNMydgklRBzl4QZrtZEqxSjCBRph6oENhEOt1NMGtrGu4IE0V7My88tiqQrugrQaf8JrytLoPi2Vx24lwB7Z64H6KlS/DHtcko6T4IShVazHIFF6K3JIL37Vb65Xs4SE5S41273EM1wBkefT3L93ktxOhZO8yMVwcHN35uJwloTTdIsWNL7Bg5H4paB0KxeboukzPLpS1vX1BOEWGVhjNhslYLmbxdhNE1UdBqWEAq1N1Rr+85Cdk8Eb8+SMTy02L4asYxjGjcRQuSizHvzG+0LPxDt0UpsZljKqjbdm/ImHsT4zsX4AnjfqifoMFeHVKF9K3nIby+h66vjoVdUsq8kYM/KHuyH7KKTpD1Im/Jl0sjkC8pCUWRN4Aj+C9O3nCTKFtjY8+aLcPx+cMgXBKvx9aPyaQ9s1/Tqb+B6URZguKr1+TNemE2kHMehK5EklVSHhikV0MmpRlh+e9EpBflcMc/x5B17Kde4ypxWLcN8/w5+FhTlRWZ3qXfPh0AuRnFuMbLGp84DUDKTi12u1ACNFISIDB6kLYah8I2UOXVTGigLpGbwNo+mkx2eUlGSAfsPSoJtTf4s2zyf+9vR8WZ8gpHlqgUjAuaQ3FLoR4LCk6k8PwX3fSYMN5ja2gPriLxY4TZj/cJ8DY0lCyLdEcni88kIWEqFo2moN9zCfx76TAaWXrTc9llmExb8OONAfr0C5cFnLhDXZIPgc+5coyVmY331o/AQIMOezNTBlyfJcDc6UKY5R4JX3VVeZ0nW+nh2I2wjD8vJ/a/JdlC7TDwRxpMi/m0RgLah/nSDOpt2ca60/hrQSwqntJkAZ2htNJFC4OuW7CF3HUg6xZIVL1H8PzaTNh26AhRmr4Dr93oJYHVplgnHoft5WJ4rcQdqfFJ2tx2G8+Vf8NipwGq4qTN7lacpOVH3CDGGvHqFzmcNdoGbrfGscVi48BtJBZu5oni6+o0kMxV5sVclsDVIc7gmXeWlJ9tIrYOrRAdqwRJk/gdzFhAW6onzfbGOrDCWUGYuz4ac7brMqvbGXRSBQfVagkT9yXg9KGMPBcWZjPnx4FbVBjBR55Y1f+e0MxpGLc3CR9ISeNMk+P4Quc4XR5WiQWtbWgt2UY3jlVjUY9u0X8qPKCnvRwPXjNCoy/DUPRdm7034cK9awkgM1UUu6/EQa29Cm9+cje1cd0Kec6niGJuDTGe8Bsut0jC9bhC4JgIaOVSRVn1Z0cmNBCGe/j+9nWIPjvyMJbm2kqg4woeE165DGboJBDuCmEmrpcBM0W3koecg/jcpoPYlxijr1Iafvgjjv8ud8OHGEqfGFH0kKrDdHExzFfRYudks2jceHcIFaP48aMaGhj0wd3T+qxztizUhMVC3IcBOjEqAkx/q/DCbVrp8Jrl4KrpTz7FtBOVuq+wtk4N0ByBM+U/JdSLMfLNjv02i0bRR2fRZ54BSw3MoMfevaSuodbsuJYF5M7+Qv42iLC/D2Nh6ZxkcjnrEEYa15GR1ROx9HwaunYJ491jvnhnTQgNGCnC1TfaMP75INVBFUaePaC8ie6Q8PMGjiuzQZs5A/DHXYOdTJeE7QHnYergLzonPhg8vLi8w6otNF5tC6iOO0pcg+uIbdAfcDdXA7M0/qZjKqBdIi3MgnExe9d3Hme9P4MhfQZMpbyShsW/pLll1mz9QyN4JP2WFO0WZXuGYqDCIpYssPHCE1VfiHHFBJw5mow9RVJoJOONWa7b6dCeYhT3bsIY114qXqfCzua8pOTdcfgx+yZKytvjuGAOz7ZSnakocKCQpUBz5ifqqHgKmjrVebdL7tAS43WwzSSUvLnUSmZINUPwHhUoceY7xqkC2mlTxJmwtiOb2BaMkbfCccJzXXZPLZkaNIqgnR9hHuvtQdolnfhxOMxhdgpMdfAirRruaDrxK7npPxXH1Sfh4BIZVBjvgddkDtAXW25hbvw3VKv8Q02a1JlV6BWafNEb/prewlrLMSj/bRDe5PId/zZFsBqOh7ZjI7TGKQJknnB5gfuF0Er7X/jmHEKq/qkn8zc0gYeEPDzI4FeZ2X9vP5rE2JeIRcz9SCSWDwVj6joDljwaS8cr1NP75/lK6FoAq0/eJ0t2ijLvb0nwjRtD8p288eTcX2RcliHG8H3CmD0iePP0QWzFEzTZoBSX327EQ1W91PAxlznWUDrP4wDs+qcE88Pm4qbDA9BzTJtxn0nDad94sOR204mpYWA2wuVdLP1BtS5uAKPM48R+5w/y3OsXTDsrDxYpZcCZJqCdWSzBmpY4souGYZgzJQLdsvVYxp48CvHdVOYFYXcXzAHDpCpyaECYVU9IgFvaEURhqScqb/1OMn+b4t41KVjzVgIPS3jje3Sl73JL0P9TE77N+U5fX+Uyy2xKzY74gFFpMTomm+OwtBAvoEWT3b4iC1PuJ0FT+R9qYx0KxtJqPOWaTlr5dwMc3XSOtHE/kV1+v6CjTRFevikGtf8F1vHABLVkwC0AAAHGelRYdE1PTCByZGtpdCAyMDIzLjA5LjUAAHicfVNLjtswDN37FLpADP7Ez3KSDIpBMQ7Qpr1D970/SirNSLMZyzQk4pF6fKS3Vs+P6/c/f9vHw9dtaw2+eCOi/WYA2N5bbdr59dvb0S73l/PTc7n9Ou4/G0rDnjG5PmNf7rf3pwfbpZ1oRw6C3k68oxGBtxPuKO4gM5jaMdwMJF5B4moA7QQ7C0Nf7uHKCrtoYGBBoytZktmNwr1PpGTSjHcUhQICWVA03AmQZQH2TAk7OXOXyh0kaJEuIVddaOrjbtUe6U7CxMqq5euk7j6h9oCGmTrXjiyZWAWJZZE4od5u5XZBTlErV5duWpSzeIMla5ROpYkJcRIMRAEdSdFJbCIR/nMNgLAsOiCT0tAWg1AXKI76e9Ljnkj05CEPkSl4uR4pqSagI6T2SQ9c+6hORMWWLiEP8QNSPypxkwdXk7JvZitPycuLnQfRYEGREVmQW+R9H8DX4/ppuh7zdr4d1zlvtWjOlKTxnBspm8NRq88RyEPT2WhJs9lMSvPZMMljzKZg2Sq9lAMXgak+SIuQMjy8CIbjI4syRWLGFMdE9FWRtf46P//V3G//AL/ZwTizd7A3AAABBnpUWHRTTUlMRVMgcmRraXQgMjAyMy4wOS41AAB4nCWQza3DQAiEW3lHW1pby/AvK6cUkCJ8dwUpPrDvhviGYeD90L29PvuNe3vuB+/92d77Ta/P33c75mk5Z/qgM6eojwOnUAJXIRYX8JhnEsm0cVCxgDZLdwseVcE5yBd0SFxVhBBTM9XytDZlYaceNNM0aTnY2JcKFhFXWQWzSrcSQp61WhBmUqyWyHKa8OTKi0ks7SiWlNQo1eA15MgI7STEE7qOClu7KscsgiKJcfBJDlDHIVkZctY+lAnqM6xj+bv71R+KbKKBLEkdHJ50dYO8pOURTDpWJiRHzZDSzFH3z7B/Iia+f396aFLjBFvx3AAAAABJRU5ErkJggg==", @@ -82,10 +90,10 @@ "my_propertymy_value" ], "text/plain": [ - "" + "" ] }, - "execution_count": 19, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -106,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 5, "id": "d5b6aa13-3951-461d-b4fc-dfcaeb169301", "metadata": {}, "outputs": [], @@ -134,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 6, "id": "2955c572-6d1d-47ff-8101-5c2781fc1c4d", "metadata": {}, "outputs": [], @@ -166,67 +174,7 @@ }, { "cell_type": "code", - "execution_count": 50, - "id": "54ff5947-3c4a-4030-8714-fc392810b1d2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
smilesmy_propertymolecule
0CN1C=NC2=C1C(=O)N(C)C(=O)N2Cmy_value/home/cas/.cache/polaris-tutorials/003/data.za...
\n", - "
" - ], - "text/plain": [ - " smiles my_property \\\n", - "0 CN1C=NC2=C1C(=O)N(C)C(=O)N2C my_value \n", - "\n", - " molecule \n", - "0 /home/cas/.cache/polaris-tutorials/003/data.za... " - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c" - ] - }, - { - "cell_type": "code", - "execution_count": 51, + "execution_count": 7, "id": "34022d65-7d1f-41ca-902d-a8385c4b6e40", "metadata": {}, "outputs": [ @@ -238,7 +186,7 @@ " 'molecule': ColumnAnnotation(is_pointer=True, modality=, description=None, user_attributes={}, dtype=dtype('O'))}" ] }, - "execution_count": 51, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -249,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 8, "id": "451b687e-34dd-4a86-9d36-b39d6247a24e", "metadata": {}, "outputs": [ @@ -257,10 +205,10 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZI0lEQVR4nO3de1hU1d4H8O8w3DHlYggaat4V0YRTjyhlgiAaCCKopGLlm+Xr8fR27Bw1FDnkBctL9qqlnkxMjEAkDK947PCGICcvKaAVISgoDigoSTAwM/v9Y0gDxYTZe7bK9/OXrplZ6zfPM8+Xvfdae22FIAggIqK2MpG7ACKiRxtjlIjIIIxRIiKDMEaJiAzCGCUiMghjlIjIIKZyF0BkdA0NOHwY589DEDBwIPz8YG4ud030CFNw3Si1LwUFCAhARQX+9CcAOHECDg7YuxcDB8pdGT2qGKPUnmg0GDoUdnbYvx8dOwJAdTXGj0d5OfLzYWYmd330SOK1UWpPDh7EuXNYt64xQwF07Ij161FQgLQ0WSujRxhjlNqT7Gx06oRnn23S6OEBBwdkZ8tUEz3yGKPUnlRWwtn5Hu3duuH6daNXQ48Jxii1J9bWqKy8R/v16+jQwejV0GOCMUrtiZsbKipQVtakUaVCWRnc3GSqiR55jFFqT4KC0KEDli9v0rhiBWxsEBIiU030yOPye2pP7OywdSumT4dKBV9fKBRIT0dKCnbsgL293MXRo4rrRql9+PZbqNUYMwYATp7Exo3Iy4MgYPBgzJ3buBSfqE0Yo9QOCAI8PHD6NBISMGWK3NXQ44bXRqkdSEnB6dNwdsaECQDQ0IDMTLlroscHY5Qed4KAmBgAWLwYVlYAEBeH55/HnDny1kWPDcYoyUMQhNraWmOMtHs3zpyBiwtmzQKAhgasWAEAL75ojNGpHWCMkgy+/vrrXr16RUVFST6STodlywAgMhIWFgDw6acoKoKrK8LCJB+d2gfGKMnAwcGhuLj4yy+/1Ol00o6UmIizZ9G9O159FQDq67FqFQBER8OEP34SB39JJANPT8+ePXuWlJRkZWVJOIxW23hVdMmSxo2Zt25FcTEGD+ZiexIRY5RkoFAoJk+eDCAhIUHCYRIScP48evZERAQAqNWIjQWAmBgeipKI+GMieUydOhVAYmKiRqORZACttvGq6NKljYeimzejtBTDhiE4WJIRqb1ijJI8hg0bNnDgwIqKim+++UaSAeLj8cMP6NMH06cDQF0d3n8fAKKjoVBIMiK1V4xRko105/UajUal73bJEpiaAsDHH+PyZbi7IzBQ9OGonWOMkmzCw8MB7NmzR61Wi9vzzp07nQ4cWB0QgGnTAKCuDqtXA0BMDA9FSXSMUZJN//79n3nmGVtbx3//+6KI3TY0NLz33nsAuoaHQ6kEUPPPf+LKFTz3HF56ScSBiPQYoySnV15JLy7+MS6un4h9xsXFXbhwoV+/flOmTAFQU1PTZ9my5c89V6ufcSISG2OU5DRxYmeFAqmpqKkRp8OGhoYVK1YAiImJUSqVADZs2HBVpdqnVFr5+oozBlFTjFGSU/fu8PTEr7+K9njjbdu2FRUVubq6hoWFAaipqVm7di2AGP06fCIJMEZJZvr9P0WZrq+vr4+NjQUQHR1tYmICYP369eXl5SNHjhyj37CZSALctplkdvUqnnoKpqa4ehW2tgZ1tWnTprlz5w4ePPjMmTMmJia3bt3q1atXRUXF0aNHR48eLVK9RM3xaJRk5uSEF1+EWo2vvjKoH7VavXLlSgAxMTH6Q9F169ZVVFR4eXkxQ0lSjFGSnyjn9Vu2bCktLR02bFhwcDCAmzdvfvjhhwCWcYKeJMYYJfmFhsLcHP/6F8rL29hDXV3dqlWrAPTr1y8iIuLq1avr1q2rrKz08fEZNWqUmLUS3YUPWCb52dlh+nTY2KANu5TU1tYeO3Zs9erVly9fBvDll1+amZmVlpaeOXMGgDF2hqZ2j1NMJL/SUmRn47nn0KNHY4tOh+RkjBiBbt3u/ZHCwuv79sXv27cvIyPj9r2knTt39vX1TUhIUCqVGo3Gz8/v0KFDRvkG1K7xaJTkl5ODyZPx3HPIzm7cCFSjweTJSElpEqNaLbKzkZaGI0dQXl5dUvIWABMTEw8PjzFjxgQEBIwcOVKhUFy8eFG/G/SSJUvk+T7UzjBG6aFgYoKiIvzzn5g9u/lLV69i/37s34/0dFRXNzba2T39+uv/8/zz7uPGjevcufPv3x8REZGVldW9e3cvLy+j1E7tHU/qSX7JyZg6FWvXIjoa58/D0RH19bCwQEoKdu3C7t24/SN1c8P48Rg/HiNGNG6Ad7eqqipnZ2eNRlNaWurk5GS0b0HtFmfq6WExZw6cnfHOO00au3WDpSXGjMGHH6K4GGfPIjYWL7zQYoYCsLOz8/f312q1iYmJUtdMBMYoPTxMTfHxx9i5E//6153GxYtx/TrS0/HWW3cmoP7QtGnTAMTHx0tQJlFzjFF6iDz/PF5+GfPmoaGhscXBAVZWre4nMDCwU6dOP/xQVlBwXdwKie7GGKWHy+rVuHIFmzcb1ImlpeWbb56tr78UH+8gUl1ELWKM0sPFyQkxMTD8Bs4xY7rX1WHnTnAOlaTGGKWHzty56NnT0E68vdGtGwoL8d13IpREdB+MUZKfoyN8fO78V6nEpk0YMwaOjm3v08SkcccTzjOR1LhulOS3YQMmTYKzs8jdnjoFDw84OuLy5fstkCIyEI9GSWYZGZg3Dx4ed2bnxeLuDldXlJc3WUFliLKysvT0dHH6oscIY5RktmgRAMydCzMz8TufOhUw+Ly+qKho/fr1vr6+3bt3DwkJqaurE6U2emzwVIfklJKC7Gw4OuIvf5Gk/5dfxvLlUCha/UFBEE6cOJGSkpKamnru3Dl9o5WVlY+PT2VlZdeuXUUulB5ljFGSjVYL/R5MUVF44glJhnByQkFBk22iGhpw6xZsbe+drQ0NyMjAv/+9Pi5udWlpqb7R3t4+ICAgKCjI39/f2tpakkLpUcYYJdns2IH8fDz9NF5/Xaoh3n4bW7bg888xfXpjy8GDmDABv/7a5Oao2locOYK0NHz1FcrLMWqUtrS01MXFZdy4cQEBAWPHjjU3N5eqRHr0MUZJHvX1eO89AFi2DJJmlKUl5s/HSy/Bzq75S+Xl2LsXqak4cgS3L3i6uWHs2Olr1oxyd3dXtOFyALU/jFGSx8aNKCqCm1vjLJB0XnwRV65g4cImN5jeuAE/P2RlQacDAKUSXl4IDkZwMHr3BuAIGLBmldoZxijJ4NYtxMYCQGxs43b30lEqsW4dfH0xcyZGjGhstLXFlSswN4eXFwICMHmy+KtWqf1gjJIMPvgA5eXw8sL48cYYztsboaGYMwcnT95pTElBr17o0MEYBdDjjetGydgqKrBuHYDGA1LjWLsWRUX43/+90zJkCDOUxMEYJWPbtGnvk0/WBwZi5EjjDdqtG5YuRUwMrl0z3qDUTvCeejKq4uLiAQMGCILy9OkLgwZ1kXq4N97A5ctISwMAjQbDhkGtRkFB8wVPRIbg0SgZVVRUlFqtDg8PM0KGNqN/SMnPPxt5WHr8MUbJePLy8uLj483NzaOiomQpwMsLM2bIMjI9zhijZDzvvvuuTqebM2dOr169jDCcIOC11xrP6G+Li4Mg8IyexMQYJSPJyclJS0vr0KHDIv2eTtJLSsLw4Zg1yzijUfvFGCUjWbhwoSAI8+fP79LFGFdFtVpERwOAp6cRRqN2jTP1ZAzx8fHTp0/v3LlzYWFhx44djTDitm2YNQt9++LcOW59T9Li74skdOnSpZSUlKSkpGPHjgGIjIw0TobW1zc+W/Qf/2CGkuR4NEri+/HHH/fs2ZOcnHzyd3dfOjo6Xrp0ycLCwggFfPQR3noLbm74/nvJ79kn4l9qEk9u7t7DhyO3b8/Ly9M3dOzY0d/fPyMjQ6VSrVq1yjgZWlODlSsBYPlyZigZA39lZLD8fERHY9AgDBnSLzU1Ly/Pzs5uxowZiYmJV65c8fPzq6ysfPLJJ2cYa8XmRx/h6lU8+ywCAowzILV3PBqlNhEEHD+O5GQkJ6O4uLHR0bGHh8fhJUtGjx5t+tslyRs3bjQ0NNTX1yuVSiPUdfMmVq8GgNjYtjyCiagNGKN0X9XV+OILnD0LrRb9+mHqVHTtiuJieHnh8uXG93TrhpAQTJoELy8rpdL3d5++efOmvb29mZnZzZs3z58/P3DgQKnr/eADVFbCxwfe3lIPRdSIU0zUstxcjB0LS0v4+MDcHJmZKCxEYiL8/eHiAlNTBAcjLAwjRjS7BllVVfX1118nJSWlp6er1Wp947JlyyIjIyWt99o19OqFX35BVhaXi5LxMEapBVotBg9Gly44dAj6qSH9zZUpKSgogFqNp55q/hGVCl99VX/wYKf9++vq6wEolcpRo0b179//448/HjZs2KlTpyQteenSwpiY3hMmIDVV0nGImmCMUguOHIGvb/PjOpUKPXrg/febPFe+ogIHDiApCQcPQqMB8OdnnvnexiYsLGzKlClOTk5qtdrR0bG6uvrnn3/u3bu3RPWWlJT069fP1fWFzz5Lc3Mzk2gUorvx2ii14ORJmJri2WebNHbpgt69G5/FUVTUOMWUkwP9H2NLS4wfj0mTPgoMNPndczgtLCxeeumlL774IiUl5Z133pGo3piYmLq6uv79OzNDyci44IlaUF0Ne/t73APUpQtu3gSA5cvxt7/h+HFYWiIgAHFxUKmQmoqICJO7nmUcEhICIDk5WaJiCwoKtm/frlQq5dqCj9ozHo1SCzp1wvXr0GiaJ6lKhe7dASA8HLW1mDQJ/v6wtr5/Z+PHj7exscnJySkpKXFxcRG92KioKI1G8/rrr/fv31/0zonuj0ej1AJ3d2i1yM9v0lhejsJCeHgAgI8P4uMREvKHGQrA2traz89PEIRUCWZ/cnNzExMTLS0tlyxZInrnRH+IMUotGD0affvi3Xf1s0aNoqNhYYHw8Db0N2nSJEhzXh8ZGanT6d58800pjnOJ/hBn6qllJ09i7Fg4O2PcOJibIyMDJ04gIQFBQW3o7JdffnF0dGxoaLhy5Yqjo6NYNf7nP/8ZPny4tbV1YWGhcXYyJWqGR6PUMg8P/PADZszA5csoKIC3N86da1uGAnjiiSe8vb21Wq245/WLFi0SBOHtt99mhpJceDRKxrNt27ZZs2aNHTv24MGDonR45MgRX19fW1vbCxcu2N21PIDIOHg0SsYTFBRkamp69OjRyspKUTrUzyktWLCAGUoyYoyS8Tg4OIwaNaqhoSGt2eM6W+/06dMRERHHjx93cnKaN2+eKOURtQ1jlIzKwPn6/Pz86OjoAQMGuLu7f/755507dw4MDLSxsRG1RqLW4fJ7MqqQkJB58+YdOnSourr6AZ/LJAhCTk7O7t27k5OTi3/b29TZ2bl3796ZmZnWD7BqlUhSPBolo+rSpYunp6darT5w4MD936nT6U6ePBkdHd23b19PT881a9YUFxc/9dRTs2fP3rt376VLl958800AKpXKKIUTtYhHo2RsISEhmZmZycnJU6ZMuftVnU6XlZWVlJSUnJx8+bedoV1cXCZOnBgWFjZixAiT3/Y21a9wKi8vN1rlRPfEBU9kbCUlJT169LCysqqoqLh9Sq7VarOzs5OSkpKSksrKyvSNPXr0CAoKCgsLGzlypOKuR4Lk5uYOGTLE1dX19hP0iGTBo1EyNhcXFw8PjxMnThw+fDgwMFCfnomJiVevXtW/oWfPnhMmTLhnel67dq1z5876f+uPRnlST7Lj0SjJIDY2dtGiRX369KmsrLy9hnTAgAGhoaGhoaFDhw5t9v6SkpI9e/YkJSXl5ORcvHixa9euAHQ6nYWFhVarVavVZmbcY5Rkw6NRksELL7zg7OxcWFgoCMKgQYMCAwMDAgK8vLyave3ChQvJycm7d+/+7rvv9H/vra2tz549q49RExMTBwcHlUp17do1Z2dnGb4GEQDGKMkiOzu7rKzM3d09Pj5+wIABzV4tLi5OTU1NSkrKysrSp6eVlZWPj09YWNjEiROfeOKJ2+/s0qWLSqVSqVSMUZIRY5RksGPHDgBRUVG/z9ALFy7onyd67NgxfYu1tbW3t3dYWFhISEiHDh3u7ke/UxQvj5K8GKNkbKdPnz579qyDg8O4ceP0LcnJyUuXLs3/bYtoOzu7wMDA0NBQPz8/C/1DSVswZMgKlSq6qmqw5EUTtYwxSsYWFxcHYNq0aebm5voWhUKRn59vZ2cXEBAQFhY2duzY2y/9kWdzc1FaKlmtRA+AMUpGpdFoEhISAMycOfN2o7+//6FDh7y9vU3vfoLefem3GOU5PcmLMUpGtW/fPpVK5erq6u7ufrtR/6SmNvSm30Sf9zGRvHhPPRmV/oz+lVdeEaU3Ho3Sw4AxSsZTWVm5f/9+U1PTadOmidIhY5QeBoxRMp5du3ap1Wo/Pz+xlnnqT+oZoyQvxigZj/6M/veTSwZydIRCgYoK6HRidUnUarynnozk3Llzrq6unTp1Kisrs7KyEqtbe3tUVeHaNTg4iNUlUetwpp6MZNeuviNHFo0alS1ihgLo0gVVVVCpGKMkGx6NkjHodOjRA6WlyMqCp6eYPR89CqUSf/oT+EAmkguPRskY0tNRWoq+fTF8uDgd1tUhIgKurli69E7jP/6Bbt3wX/8lzhBED4hTTGQMcXEAMHMm7trDvo00GiQlIToaR47caczIwKlT4vRP9OAYoyS56mqkpkKhwMsvi9yzpyfmzoVaLXK3RK3CGCXJJSbi118xejSeflrknv/2N1RXIzZW5G6JWoUxSpK7fUYvuo4dsXIlYmNRUCB+50QPiDFK0ioqwrFjsLFBSIgk/c+cCQ8P/Pd/S9I50YNgjJK0tm+HICA0FPfavb4Vbt68d7tCgU8+QUYG9uwxqH+iNmOMkoQEATt3Agaf0efno29fbN5871cHD8af/4y//x319QaNQtQ2jFGS0P/9Hy5cQI8eGDWq7Z0UF8PPDxUVOHwYLd0sEh2N2lpkZ7d9FKI2Y4yShFJTAWDGDJi09Yd27Rr8/XHlCl58EfHxLS477dgRa9ZwgxKSB+9iIgmtXo2AAPTt28aP//IL/P3x448YMgQpKbC0BIC0NOzejU2bsHkzfv9s5qlT0dAg/poqoj/Ee+pJTIGBUKuxd29j5AF4/30UFGDr1lZ3VV+PwEAcPozevZGZCScnAMjMhJ8famuxfbskK6iI2oAn9SSm3Fykp2PlyjstJSX46adW96PTYfp0HD6Mrl2Rnt6YoXl5mDABtbWYPZsZSg8RxiiJbNw4xMbi/HmDOlm6tCApCba2OHCg8Ty9qAh+fqiqQlAQNm4UpVIicTBGSWT+/vDxwZw5Lc6q/6HFixcvXz5gzJj/fP01hgwBgIoKjBuHsjKMHo2EBLTyMcxE0mKMkvjWrkV2duOK0dbatGnT8uXLTUwUc+aUenkBQHV140TT0KHYs+fOVVeihwRjlMQ3YADmz8c776CqqnUf3LVr17x58xQKxZYtW0JCQgDU1yM0FKdOoU8fHDoEW1tJCiYyBGOUJLF4MWxsEBPTio8cOXLk1Vdf1el0H3zwwWuvvQZAq8W0aUhPb5xo0j9OmehhwxglcVRWNvmvtTXWrsXGjXem6e+/Nj4nJyc4OLi+vn7hwoXz588HIAhCVNT+r76CnR0OHULPntLUTWQwxigZqrYWCxdi4EBcvdqkPTgYfn44fBgAyssxaBASEu7dQ35+/vjx42tqaiIiIlasWKFvjIyMXLHipeHDV6WlYfBgSb8BkUEYo2SQzEy4uWHVKty4gays5q9u3AhrawDYtAk//ojwcMyejZqa5m9LTk6urKwMDg7etm2bQqEAsGHDhpUrV5qZmS1a5DZihBG+B5EBBKI2qa0VFiwQlEoBEAYPFk6dEgRB+PZb4dKlJm87fVr47jtBpxM2bxasrQVAePpp4dix5r3t2LGjtrZW/++dO3eamJgoFIrPPvtM+u9BZCjGKLXFmTPC0KECIJiaCgsWCHV1D/Sp3FxhyBABEMzMhDVrqjQazd3vSUtLMzMzA7BmzRqRiyaSBmOUWqehQYiNFczNBUDo3Vv49tvWfbyuTliwQDA11Q4b5uPp6VlYWPj7V48fP25jYwMgMjJSzKKJpMQYpVbIyxM8PARAUCiE2bOFW7fa2M8331xwdnYGYGtr+8UXX+gbc3Nz7e3tAcycOVOn04lWNJHEGKP0QDQaTWxs7JAhtwChVy8hI8PQDisqKoKDg/UX6MPCwnJzc11cXABMmDChoaFBjJKJjIQb5dEf++mnn1555ZXs7Oy+fYPHjNnz/vsKAx+sdNuWLVv++te/1tTUKJVKrVY7evTo/fv3W/J+T3qkMEbpfgRB2Lp1qz7pnJyctm7dGhAQIO4QRUVF4eHh+fn5Dg4O33//vS3v96RHDWOUWnTx4sXXXnvt6NGjAMLCwj755BP9tUvR3bp16+LFi3369LGwsJCifyJJMUbp3pKSkt54442qqipHR8dPPvlk4sSJcldE9JDiXUzUnEqlCgoKmjx5clVVVWhoaF5eHjOU6D64/y01kZmZGRQUVFlZaW9vv2HDhvDwcLkrInrY8aSemrhx44abm5urq+unn37arVs3ucshegQwRqm5kpIS/RJOInoQjFEiIoNwiomIyCCMUSIigzBGiYgMwhglIjIIY5SIyCD/D9+G3rhq5bBLAAABTnpUWHRyZGtpdFBLTCByZGtpdCAyMDIzLjA5LjUAAHice79v7T0GIOBnQAA+KL+BkY0hAyTAyMzOoAFiMEMFmBkRAmCaBZ3mgNBMaBoZmQkq4GZgZGBkYmBi5mBiZmFgYeVgYmVjYGPnYGLjYODgZODgYuDi5mDi4mHg4WVgZWTgYWEQYQJqZGUEKmdlY+Pg4mFhFd8EMgqKGfiWv+A4EMzqfeAh9+T9qasm7JdQkz+wae76fb+tPuxjNbE9sOuWlf2P4MN2sicZDxgfmWl/TnKiXfyMnP0T627b/arT2h/tNG//60ds+3u8qvY36/fse1i1Z/+O9a/3u/zi3a/3X/RA1r2N+5oDM+2ntG8Fmm+w//3Jz/Y6V6QOeL8SsZcsnm5vzfh2n334ZPt9B4Udls1+su+DWIZ93K5OuwWdH+yuhX2xf28hat9UvM9eDACM1GEAYrR3BQAAAaF6VFh0TU9MIHJka2l0IDIwMjMuMDkuNQAAeJx9U0tOxDAM3fcUvgCR7TgfL5kZhBCiI8HAHdhzf2EnHZJsaOsqcZ+d52d3A7/eL6/fP/B3xcu2AeA/j6rCV0TE7Q18Aaen55cdzrfH091zvn7utw8gAUoWY/eKfbxd3+4egjM8cKCojAgPMVDhtqJAUitOwQx7c0dsAA5Sc/EVhijOaECjZ8UgWQ+ophwdEAqrzkixpBZfKXUgcvHvFBgpzsBkKTFwjdKPVBZuGYVrmYG5n51zxl4Hx3zEJM7L4aVDtZSjDi6x9iApnGZohau7qzRWnitJ7pSt+IWAuk6uSZFGUInkSEqVZyThwVWRWtGK96RCukKp1Z8qlYakGrkTkcxLUcRG1QCJPKfRw3pUJ5KXmig28RVdKhfX1WmSZl06b5N0buxqO8lYcG+S1LKc/bRflunq83a67pcxb37zmCnbQBxzYxuQMRxklsYIsFkejSazMprJZnU0jGyroynkNkvfHDQJzP4inoSk5omTYNReMinjJEZM6og0KzLX7/v7v2rr7RdbH7+0RVgL8gAAAOF6VFh0U01JTEVTIHJka2l0IDIwMjMuMDkuNQAAeJwlT0uuxDAIu8pbtlIaBUP4qOoq+86F5vAPMiuC7dhmvbSed+FZtI7nc77HOn8PrL/vcaETB9rFnQw5qZO43zV55I4urtau0Vn4ziEaG46p3EY3RNxJOs1CBywadQziROEs9TUgSK3ArSxUtYLAutkJjYLDbAfB2IsXw6wiLsRbN0UrI4tsGxaTdA0i2XJybPcYlBVi/NRCgawynSxRckbbVyDuXCdRS8fhO1lEZ2pjZMHsodH2vX6Xndc2HXWfuMX5/QcEX0W59Lht5AAAAABJRU5ErkJggg==", "text/plain": [ - "" + "" ] }, - "execution_count": 54, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -292,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 9, "id": "18beb7e0-95f2-4fd2-917d-8d4bcceb65af", "metadata": {}, "outputs": [ @@ -300,10 +248,10 @@ "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcIAAACWCAIAAADCEh9HAAAABmJLR0QA/wD/AP+gvaeTAAAZI0lEQVR4nO3de1hU1d4H8O8w3DHlYggaat4V0YRTjyhlgiAaCCKopGLlm+Xr8fR27Bw1FDnkBctL9qqlnkxMjEAkDK947PCGICcvKaAVISgoDigoSTAwM/v9Y0gDxYTZe7bK9/OXrplZ6zfPM8+Xvfdae22FIAggIqK2MpG7ACKiRxtjlIjIIIxRIiKDMEaJiAzCGCUiMghjlIjIIKZyF0BkdA0NOHwY589DEDBwIPz8YG4ud030CFNw3Si1LwUFCAhARQX+9CcAOHECDg7YuxcDB8pdGT2qGKPUnmg0GDoUdnbYvx8dOwJAdTXGj0d5OfLzYWYmd330SOK1UWpPDh7EuXNYt64xQwF07Ij161FQgLQ0WSujRxhjlNqT7Gx06oRnn23S6OEBBwdkZ8tUEz3yGKPUnlRWwtn5Hu3duuH6daNXQ48Jxii1J9bWqKy8R/v16+jQwejV0GOCMUrtiZsbKipQVtakUaVCWRnc3GSqiR55jFFqT4KC0KEDli9v0rhiBWxsEBIiU030yOPye2pP7OywdSumT4dKBV9fKBRIT0dKCnbsgL293MXRo4rrRql9+PZbqNUYMwYATp7Exo3Iy4MgYPBgzJ3buBSfqE0Yo9QOCAI8PHD6NBISMGWK3NXQ44bXRqkdSEnB6dNwdsaECQDQ0IDMTLlroscHY5Qed4KAmBgAWLwYVlYAEBeH55/HnDny1kWPDcYoyUMQhNraWmOMtHs3zpyBiwtmzQKAhgasWAEAL75ojNGpHWCMkgy+/vrrXr16RUVFST6STodlywAgMhIWFgDw6acoKoKrK8LCJB+d2gfGKMnAwcGhuLj4yy+/1Ol00o6UmIizZ9G9O159FQDq67FqFQBER8OEP34SB39JJANPT8+ePXuWlJRkZWVJOIxW23hVdMmSxo2Zt25FcTEGD+ZiexIRY5RkoFAoJk+eDCAhIUHCYRIScP48evZERAQAqNWIjQWAmBgeipKI+GMieUydOhVAYmKiRqORZACttvGq6NKljYeimzejtBTDhiE4WJIRqb1ijJI8hg0bNnDgwIqKim+++UaSAeLj8cMP6NMH06cDQF0d3n8fAKKjoVBIMiK1V4xRko105/UajUal73bJEpiaAsDHH+PyZbi7IzBQ9OGonWOMkmzCw8MB7NmzR61Wi9vzzp07nQ4cWB0QgGnTAKCuDqtXA0BMDA9FSXSMUZJN//79n3nmGVtbx3//+6KI3TY0NLz33nsAuoaHQ6kEUPPPf+LKFTz3HF56ScSBiPQYoySnV15JLy7+MS6un4h9xsXFXbhwoV+/flOmTAFQU1PTZ9my5c89V6ufcSISG2OU5DRxYmeFAqmpqKkRp8OGhoYVK1YAiImJUSqVADZs2HBVpdqnVFr5+oozBlFTjFGSU/fu8PTEr7+K9njjbdu2FRUVubq6hoWFAaipqVm7di2AGP06fCIJMEZJZvr9P0WZrq+vr4+NjQUQHR1tYmICYP369eXl5SNHjhyj37CZSALctplkdvUqnnoKpqa4ehW2tgZ1tWnTprlz5w4ePPjMmTMmJia3bt3q1atXRUXF0aNHR48eLVK9RM3xaJRk5uSEF1+EWo2vvjKoH7VavXLlSgAxMTH6Q9F169ZVVFR4eXkxQ0lSjFGSnyjn9Vu2bCktLR02bFhwcDCAmzdvfvjhhwCWcYKeJMYYJfmFhsLcHP/6F8rL29hDXV3dqlWrAPTr1y8iIuLq1avr1q2rrKz08fEZNWqUmLUS3YUPWCb52dlh+nTY2KANu5TU1tYeO3Zs9erVly9fBvDll1+amZmVlpaeOXMGgDF2hqZ2j1NMJL/SUmRn47nn0KNHY4tOh+RkjBiBbt3u/ZHCwuv79sXv27cvIyPj9r2knTt39vX1TUhIUCqVGo3Gz8/v0KFDRvkG1K7xaJTkl5ODyZPx3HPIzm7cCFSjweTJSElpEqNaLbKzkZaGI0dQXl5dUvIWABMTEw8PjzFjxgQEBIwcOVKhUFy8eFG/G/SSJUvk+T7UzjBG6aFgYoKiIvzzn5g9u/lLV69i/37s34/0dFRXNzba2T39+uv/8/zz7uPGjevcufPv3x8REZGVldW9e3cvLy+j1E7tHU/qSX7JyZg6FWvXIjoa58/D0RH19bCwQEoKdu3C7t24/SN1c8P48Rg/HiNGNG6Ad7eqqipnZ2eNRlNaWurk5GS0b0HtFmfq6WExZw6cnfHOO00au3WDpSXGjMGHH6K4GGfPIjYWL7zQYoYCsLOz8/f312q1iYmJUtdMBMYoPTxMTfHxx9i5E//6153GxYtx/TrS0/HWW3cmoP7QtGnTAMTHx0tQJlFzjFF6iDz/PF5+GfPmoaGhscXBAVZWre4nMDCwU6dOP/xQVlBwXdwKie7GGKWHy+rVuHIFmzcb1ImlpeWbb56tr78UH+8gUl1ELWKM0sPFyQkxMTD8Bs4xY7rX1WHnTnAOlaTGGKWHzty56NnT0E68vdGtGwoL8d13IpREdB+MUZKfoyN8fO78V6nEpk0YMwaOjm3v08SkcccTzjOR1LhulOS3YQMmTYKzs8jdnjoFDw84OuLy5fstkCIyEI9GSWYZGZg3Dx4ed2bnxeLuDldXlJc3WUFliLKysvT0dHH6oscIY5RktmgRAMydCzMz8TufOhUw+Ly+qKho/fr1vr6+3bt3DwkJqaurE6U2emzwVIfklJKC7Gw4OuIvf5Gk/5dfxvLlUCha/UFBEE6cOJGSkpKamnru3Dl9o5WVlY+PT2VlZdeuXUUulB5ljFGSjVYL/R5MUVF44glJhnByQkFBk22iGhpw6xZsbe+drQ0NyMjAv/+9Pi5udWlpqb7R3t4+ICAgKCjI39/f2tpakkLpUcYYJdns2IH8fDz9NF5/Xaoh3n4bW7bg888xfXpjy8GDmDABv/7a5Oao2locOYK0NHz1FcrLMWqUtrS01MXFZdy4cQEBAWPHjjU3N5eqRHr0MUZJHvX1eO89AFi2DJJmlKUl5s/HSy/Bzq75S+Xl2LsXqak4cgS3L3i6uWHs2Olr1oxyd3dXtOFyALU/jFGSx8aNKCqCm1vjLJB0XnwRV65g4cImN5jeuAE/P2RlQacDAKUSXl4IDkZwMHr3BuAIGLBmldoZxijJ4NYtxMYCQGxs43b30lEqsW4dfH0xcyZGjGhstLXFlSswN4eXFwICMHmy+KtWqf1gjJIMPvgA5eXw8sL48cYYztsboaGYMwcnT95pTElBr17o0MEYBdDjjetGydgqKrBuHYDGA1LjWLsWRUX43/+90zJkCDOUxMEYJWPbtGnvk0/WBwZi5EjjDdqtG5YuRUwMrl0z3qDUTvCeejKq4uLiAQMGCILy9OkLgwZ1kXq4N97A5ctISwMAjQbDhkGtRkFB8wVPRIbg0SgZVVRUlFqtDg8PM0KGNqN/SMnPPxt5WHr8MUbJePLy8uLj483NzaOiomQpwMsLM2bIMjI9zhijZDzvvvuuTqebM2dOr169jDCcIOC11xrP6G+Li4Mg8IyexMQYJSPJyclJS0vr0KHDIv2eTtJLSsLw4Zg1yzijUfvFGCUjWbhwoSAI8+fP79LFGFdFtVpERwOAp6cRRqN2jTP1ZAzx8fHTp0/v3LlzYWFhx44djTDitm2YNQt9++LcOW59T9Li74skdOnSpZSUlKSkpGPHjgGIjIw0TobW1zc+W/Qf/2CGkuR4NEri+/HHH/fs2ZOcnHzyd3dfOjo6Xrp0ycLCwggFfPQR3noLbm74/nvJ79kn4l9qEk9u7t7DhyO3b8/Ly9M3dOzY0d/fPyMjQ6VSrVq1yjgZWlODlSsBYPlyZigZA39lZLD8fERHY9AgDBnSLzU1Ly/Pzs5uxowZiYmJV65c8fPzq6ysfPLJJ2cYa8XmRx/h6lU8+ywCAowzILV3PBqlNhEEHD+O5GQkJ6O4uLHR0bGHh8fhJUtGjx5t+tslyRs3bjQ0NNTX1yuVSiPUdfMmVq8GgNjYtjyCiagNGKN0X9XV+OILnD0LrRb9+mHqVHTtiuJieHnh8uXG93TrhpAQTJoELy8rpdL3d5++efOmvb29mZnZzZs3z58/P3DgQKnr/eADVFbCxwfe3lIPRdSIU0zUstxcjB0LS0v4+MDcHJmZKCxEYiL8/eHiAlNTBAcjLAwjRjS7BllVVfX1118nJSWlp6er1Wp947JlyyIjIyWt99o19OqFX35BVhaXi5LxMEapBVotBg9Gly44dAj6qSH9zZUpKSgogFqNp55q/hGVCl99VX/wYKf9++vq6wEolcpRo0b179//448/HjZs2KlTpyQteenSwpiY3hMmIDVV0nGImmCMUguOHIGvb/PjOpUKPXrg/febPFe+ogIHDiApCQcPQqMB8OdnnvnexiYsLGzKlClOTk5qtdrR0bG6uvrnn3/u3bu3RPWWlJT069fP1fWFzz5Lc3Mzk2gUorvx2ii14ORJmJri2WebNHbpgt69G5/FUVTUOMWUkwP9H2NLS4wfj0mTPgoMNPndczgtLCxeeumlL774IiUl5Z133pGo3piYmLq6uv79OzNDyci44IlaUF0Ne/t73APUpQtu3gSA5cvxt7/h+HFYWiIgAHFxUKmQmoqICJO7nmUcEhICIDk5WaJiCwoKtm/frlQq5dqCj9ozHo1SCzp1wvXr0GiaJ6lKhe7dASA8HLW1mDQJ/v6wtr5/Z+PHj7exscnJySkpKXFxcRG92KioKI1G8/rrr/fv31/0zonuj0ej1AJ3d2i1yM9v0lhejsJCeHgAgI8P4uMREvKHGQrA2traz89PEIRUCWZ/cnNzExMTLS0tlyxZInrnRH+IMUotGD0affvi3Xf1s0aNoqNhYYHw8Db0N2nSJEhzXh8ZGanT6d58800pjnOJ/hBn6qllJ09i7Fg4O2PcOJibIyMDJ04gIQFBQW3o7JdffnF0dGxoaLhy5Yqjo6NYNf7nP/8ZPny4tbV1YWGhcXYyJWqGR6PUMg8P/PADZszA5csoKIC3N86da1uGAnjiiSe8vb21Wq245/WLFi0SBOHtt99mhpJceDRKxrNt27ZZs2aNHTv24MGDonR45MgRX19fW1vbCxcu2N21PIDIOHg0SsYTFBRkamp69OjRyspKUTrUzyktWLCAGUoyYoyS8Tg4OIwaNaqhoSGt2eM6W+/06dMRERHHjx93cnKaN2+eKOURtQ1jlIzKwPn6/Pz86OjoAQMGuLu7f/755507dw4MDLSxsRG1RqLW4fJ7MqqQkJB58+YdOnSourr6AZ/LJAhCTk7O7t27k5OTi3/b29TZ2bl3796ZmZnWD7BqlUhSPBolo+rSpYunp6darT5w4MD936nT6U6ePBkdHd23b19PT881a9YUFxc/9dRTs2fP3rt376VLl958800AKpXKKIUTtYhHo2RsISEhmZmZycnJU6ZMuftVnU6XlZWVlJSUnJx8+bedoV1cXCZOnBgWFjZixAiT3/Y21a9wKi8vN1rlRPfEBU9kbCUlJT169LCysqqoqLh9Sq7VarOzs5OSkpKSksrKyvSNPXr0CAoKCgsLGzlypOKuR4Lk5uYOGTLE1dX19hP0iGTBo1EyNhcXFw8PjxMnThw+fDgwMFCfnomJiVevXtW/oWfPnhMmTLhnel67dq1z5876f+uPRnlST7Lj0SjJIDY2dtGiRX369KmsrLy9hnTAgAGhoaGhoaFDhw5t9v6SkpI9e/YkJSXl5ORcvHixa9euAHQ6nYWFhVarVavVZmbcY5Rkw6NRksELL7zg7OxcWFgoCMKgQYMCAwMDAgK8vLyave3ChQvJycm7d+/+7rvv9H/vra2tz549q49RExMTBwcHlUp17do1Z2dnGb4GEQDGKMkiOzu7rKzM3d09Pj5+wIABzV4tLi5OTU1NSkrKysrSp6eVlZWPj09YWNjEiROfeOKJ2+/s0qWLSqVSqVSMUZIRY5RksGPHDgBRUVG/z9ALFy7onyd67NgxfYu1tbW3t3dYWFhISEiHDh3u7ke/UxQvj5K8GKNkbKdPnz579qyDg8O4ceP0LcnJyUuXLs3/bYtoOzu7wMDA0NBQPz8/C/1DSVswZMgKlSq6qmqw5EUTtYwxSsYWFxcHYNq0aebm5voWhUKRn59vZ2cXEBAQFhY2duzY2y/9kWdzc1FaKlmtRA+AMUpGpdFoEhISAMycOfN2o7+//6FDh7y9vU3vfoLefem3GOU5PcmLMUpGtW/fPpVK5erq6u7ufrtR/6SmNvSm30Sf9zGRvHhPPRmV/oz+lVdeEaU3Ho3Sw4AxSsZTWVm5f/9+U1PTadOmidIhY5QeBoxRMp5du3ap1Wo/Pz+xlnnqT+oZoyQvxigZj/6M/veTSwZydIRCgYoK6HRidUnUarynnozk3Llzrq6unTp1Kisrs7KyEqtbe3tUVeHaNTg4iNUlUetwpp6MZNeuviNHFo0alS1ihgLo0gVVVVCpGKMkGx6NkjHodOjRA6WlyMqCp6eYPR89CqUSf/oT+EAmkguPRskY0tNRWoq+fTF8uDgd1tUhIgKurli69E7jP/6Bbt3wX/8lzhBED4hTTGQMcXEAMHMm7trDvo00GiQlIToaR47caczIwKlT4vRP9OAYoyS56mqkpkKhwMsvi9yzpyfmzoVaLXK3RK3CGCXJJSbi118xejSeflrknv/2N1RXIzZW5G6JWoUxSpK7fUYvuo4dsXIlYmNRUCB+50QPiDFK0ioqwrFjsLFBSIgk/c+cCQ8P/Pd/S9I50YNgjJK0tm+HICA0FPfavb4Vbt68d7tCgU8+QUYG9uwxqH+iNmOMkoQEATt3Agaf0efno29fbN5871cHD8af/4y//x319QaNQtQ2jFGS0P/9Hy5cQI8eGDWq7Z0UF8PPDxUVOHwYLd0sEh2N2lpkZ7d9FKI2Y4yShFJTAWDGDJi09Yd27Rr8/XHlCl58EfHxLS477dgRa9ZwgxKSB+9iIgmtXo2AAPTt28aP//IL/P3x448YMgQpKbC0BIC0NOzejU2bsHkzfv9s5qlT0dAg/poqoj/Ee+pJTIGBUKuxd29j5AF4/30UFGDr1lZ3VV+PwEAcPozevZGZCScnAMjMhJ8famuxfbskK6iI2oAn9SSm3Fykp2PlyjstJSX46adW96PTYfp0HD6Mrl2Rnt6YoXl5mDABtbWYPZsZSg8RxiiJbNw4xMbi/HmDOlm6tCApCba2OHCg8Ty9qAh+fqiqQlAQNm4UpVIicTBGSWT+/vDxwZw5Lc6q/6HFixcvXz5gzJj/fP01hgwBgIoKjBuHsjKMHo2EBLTyMcxE0mKMkvjWrkV2duOK0dbatGnT8uXLTUwUc+aUenkBQHV140TT0KHYs+fOVVeihwRjlMQ3YADmz8c776CqqnUf3LVr17x58xQKxZYtW0JCQgDU1yM0FKdOoU8fHDoEW1tJCiYyBGOUJLF4MWxsEBPTio8cOXLk1Vdf1el0H3zwwWuvvQZAq8W0aUhPb5xo0j9OmehhwxglcVRWNvmvtTXWrsXGjXem6e+/Nj4nJyc4OLi+vn7hwoXz588HIAhCVNT+r76CnR0OHULPntLUTWQwxigZqrYWCxdi4EBcvdqkPTgYfn44fBgAyssxaBASEu7dQ35+/vjx42tqaiIiIlasWKFvjIyMXLHipeHDV6WlYfBgSb8BkUEYo2SQzEy4uWHVKty4gays5q9u3AhrawDYtAk//ojwcMyejZqa5m9LTk6urKwMDg7etm2bQqEAsGHDhpUrV5qZmS1a5DZihBG+B5EBBKI2qa0VFiwQlEoBEAYPFk6dEgRB+PZb4dKlJm87fVr47jtBpxM2bxasrQVAePpp4dix5r3t2LGjtrZW/++dO3eamJgoFIrPPvtM+u9BZCjGKLXFmTPC0KECIJiaCgsWCHV1D/Sp3FxhyBABEMzMhDVrqjQazd3vSUtLMzMzA7BmzRqRiyaSBmOUWqehQYiNFczNBUDo3Vv49tvWfbyuTliwQDA11Q4b5uPp6VlYWPj7V48fP25jYwMgMjJSzKKJpMQYpVbIyxM8PARAUCiE2bOFW7fa2M8331xwdnYGYGtr+8UXX+gbc3Nz7e3tAcycOVOn04lWNJHEGKP0QDQaTWxs7JAhtwChVy8hI8PQDisqKoKDg/UX6MPCwnJzc11cXABMmDChoaFBjJKJjIQb5dEf++mnn1555ZXs7Oy+fYPHjNnz/vsKAx+sdNuWLVv++te/1tTUKJVKrVY7evTo/fv3W/J+T3qkMEbpfgRB2Lp1qz7pnJyctm7dGhAQIO4QRUVF4eHh+fn5Dg4O33//vS3v96RHDWOUWnTx4sXXXnvt6NGjAMLCwj755BP9tUvR3bp16+LFi3369LGwsJCifyJJMUbp3pKSkt54442qqipHR8dPPvlk4sSJcldE9JDiXUzUnEqlCgoKmjx5clVVVWhoaF5eHjOU6D64/y01kZmZGRQUVFlZaW9vv2HDhvDwcLkrInrY8aSemrhx44abm5urq+unn37arVs3ucshegQwRqm5kpIS/RJOInoQjFEiIoNwiomIyCCMUSIigzBGiYgMwhglIjIIY5SIyCD/D9+G3rhq5bBLAAABTnpUWHRyZGtpdFBLTCByZGtpdCAyMDIzLjA5LjUAAHice79v7T0GIOBnQAA+KL+BkY0hAyTAyMzOoAFiMEMFmBkRAmCaBZ3mgNBMaBoZmQkq4GZgZGBkYmBi5mBiZmFgYeVgYmVjYGPnYGLjYODgZODgYuDi5mDi4mHg4WVgZWTgYWEQYQJqZGUEKmdlY+Pg4mFhFd8EMgqKGfiWv+A4EMzqfeAh9+T9qasm7JdQkz+wae76fb+tPuxjNbE9sOuWlf2P4MN2sicZDxgfmWl/TnKiXfyMnP0T627b/arT2h/tNG//60ds+3u8qvY36/fse1i1Z/+O9a/3u/zi3a/3X/RA1r2N+5oDM+2ntG8Fmm+w//3Jz/Y6V6QOeL8SsZcsnm5vzfh2n334ZPt9B4Udls1+su+DWIZ93K5OuwWdH+yuhX2xf28hat9UvM9eDACM1GEAYrR3BQAAAaF6VFh0TU9MIHJka2l0IDIwMjMuMDkuNQAAeJx9U0tOxDAM3fcUvgCR7TgfL5kZhBCiI8HAHdhzf2EnHZJsaOsqcZ+d52d3A7/eL6/fP/B3xcu2AeA/j6rCV0TE7Q18Aaen55cdzrfH091zvn7utw8gAUoWY/eKfbxd3+4egjM8cKCojAgPMVDhtqJAUitOwQx7c0dsAA5Sc/EVhijOaECjZ8UgWQ+ophwdEAqrzkixpBZfKXUgcvHvFBgpzsBkKTFwjdKPVBZuGYVrmYG5n51zxl4Hx3zEJM7L4aVDtZSjDi6x9iApnGZohau7qzRWnitJ7pSt+IWAuk6uSZFGUInkSEqVZyThwVWRWtGK96RCukKp1Z8qlYakGrkTkcxLUcRG1QCJPKfRw3pUJ5KXmig28RVdKhfX1WmSZl06b5N0buxqO8lYcG+S1LKc/bRflunq83a67pcxb37zmCnbQBxzYxuQMRxklsYIsFkejSazMprJZnU0jGyroynkNkvfHDQJzP4inoSk5omTYNReMinjJEZM6og0KzLX7/v7v2rr7RdbH7+0RVgL8gAAAOF6VFh0U01JTEVTIHJka2l0IDIwMjMuMDkuNQAAeJwlT0uuxDAIu8pbtlIaBUP4qOoq+86F5vAPMiuC7dhmvbSed+FZtI7nc77HOn8PrL/vcaETB9rFnQw5qZO43zV55I4urtau0Vn4ziEaG46p3EY3RNxJOs1CBywadQziROEs9TUgSK3ArSxUtYLAutkJjYLDbAfB2IsXw6wiLsRbN0UrI4tsGxaTdA0i2XJybPcYlBVi/NRCgawynSxRckbbVyDuXCdRS8fhO1lEZ2pjZMHsodH2vX6Xndc2HXWfuMX5/QcEX0W59Lht5AAAAABJRU5ErkJggg==", "text/plain": [ - "" + "" ] }, - "execution_count": 59, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -330,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 10, "id": "35b6e2cb-3b45-4944-903d-7da81ff1e7a4", "metadata": {}, "outputs": [ @@ -338,7 +286,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-03-14 15:26:05.284\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m99\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n" + "\u001b[32m2024-03-14 15:33:36.569\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m100\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n" ] } ], @@ -369,7 +317,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 11, "id": "dbd94922-a9b9-4096-b42b-4e593581b947", "metadata": {}, "outputs": [ @@ -381,10 +329,10 @@ "my_propertymy_value" ], "text/plain": [ - "" + "" ] }, - "execution_count": 71, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -395,7 +343,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 12, "id": "2b12c7c0-23be-4286-8dca-23d0e7a606cf", "metadata": {}, "outputs": [ @@ -442,7 +390,7 @@ "0 /home/cas/.cache/polaris-tutorials/003/data2.z... " ] }, - "execution_count": 72, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -463,7 +411,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 13, "id": "ef15bf98-f301-465d-9e93-2531f9f1f98c", "metadata": {}, "outputs": [ @@ -471,8 +419,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-03-14 15:29:39.280\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m99\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n", - "\u001b[32m2024-03-14 15:29:39.284\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m99\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n" + "\u001b[32m2024-03-14 15:33:36.611\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m100\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n", + "\u001b[32m2024-03-14 15:33:36.614\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mpolaris.dataset._factory\u001b[0m:\u001b[36mregister_converter\u001b[0m:\u001b[36m100\u001b[0m - \u001b[1mYou are overwriting the converter for the sdf extension.\u001b[0m\n" ] } ], @@ -493,7 +441,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 14, "id": "65960c85-ee0d-4d37-b50d-b1c8ba1cec64", "metadata": {}, "outputs": [ @@ -547,7 +495,7 @@ "0 /home/cas/.cache/polaris-tutorials/003/data3.z... " ] }, - "execution_count": 80, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py index 488ecbec..0953ed69 100644 --- a/polaris/dataset/_factory.py +++ b/polaris/dataset/_factory.py @@ -61,6 +61,7 @@ def __init__(self, zarr_root_path: Optional[str] = None) -> None: zarr_root_path: The root path of the zarr hierarchy. If you want to use pointer columns, this arguments needs to be passed. """ + self._converters: Dict[str, Converter] = {} self.reset(zarr_root_path=zarr_root_path) @property From f9e4b872917a84e6d18b06b05bbfabffb798058f Mon Sep 17 00:00:00 2001 From: cwognum Date: Thu, 14 Mar 2024 16:46:49 -0400 Subject: [PATCH 7/9] Added test cases --- polaris/dataset/_factory.py | 2 +- tests/conftest.py | 38 +++++++++++----- tests/test_dataset.py | 36 +++++++++------ tests/test_factory.py | 75 ++++++++++++++++++++++++++++++++ tests/test_to_zarr_converters.py | 8 ---- 5 files changed, 126 insertions(+), 33 deletions(-) create mode 100644 tests/test_factory.py delete mode 100644 tests/test_to_zarr_converters.py diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py index 0953ed69..3e5d16b6 100644 --- a/polaris/dataset/_factory.py +++ b/polaris/dataset/_factory.py @@ -178,7 +178,7 @@ def add_columns( adapters = {**self._adapters, **adapters} if merge_on is not None: - self.reset() + self.reset(self._zarr_root_path) for name, series in df.items(): annotation = annotations.get(name) diff --git a/tests/conftest.py b/tests/conftest.py index 1105c151..42df3b5e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,14 +12,6 @@ from polaris.utils.types import HubOwner, License -def _get_zarr_archive(tmp_path): - tmp_path = fs.join(str(tmp_path), "data.zarr") - root = zarr.open_group(tmp_path, mode="w") - root.array("A", data=np.random.random((100, 2048))) - root.array("B", data=np.random.random((100, 2048))) - return tmp_path - - @pytest.fixture(scope="module") def test_data(): data = dm.data.freesolv()[:100] @@ -29,6 +21,28 @@ def test_data(): return data +@pytest.fixture(scope="module") +def caffeine(): + # Let's generate a toy dataset with a single molecule + smiles = "Cn1cnc2c1c(=O)n(C)c(=O)n2C" + mol = dm.to_mol(smiles) + + # We will generate 3D conformers for this molecule with some conformers + # NOTE (cwognum): We only generate a single conformer, because dm.to_sdf() only saves one. + mol = dm.conformers.generate(mol, align_conformers=True, n_confs=1) + + # Let's also set a molecular property + mol.SetProp("my_property", "my_value") + return mol + + +@pytest.fixture(scope="module") +def sdf_file(tmp_path_factory, caffeine): + path = tmp_path_factory.mktemp("data") / "caffeine.sdf" + dm.to_sdf(caffeine, path) + return path + + @pytest.fixture(scope="module") def test_org_owner(): return HubOwner(organizationId="test-organization", slug="test-organization") @@ -55,8 +69,12 @@ def test_dataset(test_data, test_org_owner): @pytest.fixture(scope="function") -def test_zarr_archive_single_array(tmp_path): - return _get_zarr_archive(tmp_path) +def zarr_archive(tmp_path): + tmp_path = fs.join(str(tmp_path), "data.zarr") + root = zarr.open_group(tmp_path, mode="w") + root.array("A", data=np.random.random((100, 2048))) + root.array("B", data=np.random.random((100, 2048))) + return tmp_path @pytest.fixture(scope="function") diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 5a05b0f5..9eab85e2 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -34,7 +34,9 @@ def _equality_test(dataset_1, dataset_2): return True -def test_load_data(tmp_path): +@pytest.mark.parametrize("with_caching", [True, False]) +@pytest.mark.parametrize("with_slice", [True, False]) +def test_load_data(tmp_path, with_slice, with_caching): """Test accessing the data, specifically whether pointer columns are properly handled.""" # Dummy data (could e.g. be a 3D structure or Image) @@ -46,19 +48,25 @@ def test_load_data(tmp_path): root = zarr.open(path, "w") root.array("A", data=arr) - path = f"{path}/A#0" + path = f"{path}/A#0:5" if with_slice else f"{path}/A#0" table = pd.DataFrame({"A": [path]}, index=[0]) dataset = Dataset(table=table, annotations={"A": {"is_pointer": True}}) - # Without caching + if with_caching: + dataset.cache(tmpdir) data = dataset.get_data(row=0, col="A") - assert (data == arr[0]).all() - # With caching - dataset.cache(tmpdir) - data = dataset.get_data(row=0, col="A") - assert (data == arr[0]).all() + if with_slice: + assert isinstance(data, tuple) + assert len(data) == 5 + + for i, d in enumerate(data): + assert (d == arr[i]).all() + + else: + data = dataset.get_data(row=0, col="A") + assert (data == arr[0]).all() def test_dataset_checksum(test_dataset): @@ -103,9 +111,9 @@ def _check_for_failure(_kwargs): assert dataset.md5sum is not None -def test_dataset_from_zarr(test_zarr_archive_single_array, tmpdir): +def test_dataset_from_zarr(zarr_archive, tmpdir): """Test whether loading works when the zarr archive contains a single array or multiple arrays.""" - archive = test_zarr_archive_single_array + archive = zarr_archive dataset = create_dataset_from_file(archive, tmpdir.join("data")) assert len(dataset.table) == 100 @@ -127,7 +135,7 @@ def test_dataset_from_json(test_dataset, tmpdir): assert _equality_test(test_dataset, new_dataset) -def test_dataset_from_zarr_to_json_and_back(test_zarr_archive_single_array, tmpdir): +def test_dataset_from_zarr_to_json_and_back(zarr_archive, tmpdir): """ Test whether a dataset with pointer columns, instantiated from a zarr archive, can be saved to and loaded from json. @@ -136,7 +144,7 @@ def test_dataset_from_zarr_to_json_and_back(test_zarr_archive_single_array, tmpd json_dir = tmpdir.join("json") zarr_dir = tmpdir.join("zarr") - archive = test_zarr_archive_single_array + archive = zarr_archive dataset = create_dataset_from_file(archive, zarr_dir) path = dataset.to_json(json_dir) @@ -147,9 +155,9 @@ def test_dataset_from_zarr_to_json_and_back(test_zarr_archive_single_array, tmpd assert _equality_test(dataset, new_dataset) -def test_dataset_caching(test_zarr_archive_single_array, tmpdir): +def test_dataset_caching(zarr_archive, tmpdir): """Test whether the dataset remains the same after caching.""" - archive = test_zarr_archive_single_array + archive = zarr_archive original_dataset = create_dataset_from_file(archive, tmpdir.join("original1")) cached_dataset = create_dataset_from_file(archive, tmpdir.join("original2")) diff --git a/tests/test_factory.py b/tests/test_factory.py new file mode 100644 index 00000000..dde155ff --- /dev/null +++ b/tests/test_factory.py @@ -0,0 +1,75 @@ +import datamol as dm +import pandas as pd +import pytest + +from polaris.dataset import DatasetFactory, create_dataset_from_file +from polaris.dataset.converters import SDFConverter, ZarrConverter + + +def _check_dataset(dataset, ground_truth, mol_props_as_col): + assert len(dataset) == 1 + + mol = dataset.get_data(row=0, col="molecule") + + assert isinstance(mol, dm.Mol) + + if mol_props_as_col: + assert not mol.HasProp("my_property") + v = dataset.get_data(row=0, col="my_property") + assert v == ground_truth.GetProp("my_property") + + else: + assert mol.HasProp("my_property") + assert mol.GetProp("my_property") == ground_truth.GetProp("my_property") + assert "my_property" not in dataset.columns + + +def test_sdf_zarr_conversion(sdf_file, caffeine, tmpdir): + """Test conversion between SDF and Zarr with utility function""" + dataset = create_dataset_from_file(sdf_file, tmpdir.join("archive.zarr")) + _check_dataset(dataset, caffeine, True) + + +@pytest.mark.parametrize("mol_props_as_col", [True, False]) +def test_factory_sdf_with_prop_as_col(sdf_file, caffeine, tmpdir, mol_props_as_col): + """Test conversion between SDF and Zarr with factory pattern""" + + factory = DatasetFactory(tmpdir.join("archive.zarr")) + + converter = SDFConverter(mol_prop_as_cols=mol_props_as_col) + factory.register_converter("sdf", converter) + + factory.add_from_file(sdf_file) + dataset = factory.build() + + _check_dataset(dataset, caffeine, mol_props_as_col) + + +def test_zarr_to_zarr_conversion(zarr_archive, tmpdir): + """Test conversion between Zarr and Zarr with utility function""" + dataset = create_dataset_from_file(zarr_archive, tmpdir.join("archive.zarr")) + assert len(dataset) == 100 + assert len(dataset.columns) == 2 + assert all(c in dataset.columns for c in ["A", "B"]) + assert all(dataset.annotations[c].is_pointer for c in ["A", "B"]) + assert dataset.get_data(row=0, col="A").shape == (2048,) + + +def test_zarr_with_factory_pattern(zarr_archive, tmpdir): + """Test conversion between Zarr and Zarr with factory pattern""" + + factory = DatasetFactory(tmpdir.join("archive.zarr")) + converter = ZarrConverter() + factory.register_converter("zarr", converter) + factory.add_from_file(zarr_archive) + + factory.add_column(pd.Series([1, 2, 3, 4] * 25, name="C")) + + df = pd.DataFrame({"C": [1, 2, 3, 4], "D": ["W", "X", "Y", "Z"]}) + factory.add_columns(df, merge_on="C") + + dataset = factory.build() + assert len(dataset) == 100 + assert len(dataset.columns) == 4 + assert all(c in dataset.columns for c in ["A", "B", "C", "D"]) + assert dataset.table["C"].apply({1: "W", 2: "X", 3: "Y", 4: "Z"}.get).equals(dataset.table["D"]) diff --git a/tests/test_to_zarr_converters.py b/tests/test_to_zarr_converters.py deleted file mode 100644 index 84f11895..00000000 --- a/tests/test_to_zarr_converters.py +++ /dev/null @@ -1,8 +0,0 @@ -import datamol as dm - - -def test_sdf_bytestring_compat(tmpdir): - "CCC(=O)F", "CC=C(O)F" - dm.Mol - print(tmpdir) - pass From 52cee53eba9780ff6ea249913f75b61cf41aa8c9 Mon Sep 17 00:00:00 2001 From: cwognum Date: Thu, 14 Mar 2024 17:13:18 -0400 Subject: [PATCH 8/9] Ruff formatting --- polaris/utils/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polaris/utils/types.py b/polaris/utils/types.py index 0cb3ba92..94e50982 100644 --- a/polaris/utils/types.py +++ b/polaris/utils/types.py @@ -124,9 +124,9 @@ class License(BaseModel): Else it is required to manually specify this. """ - SPDX_LICENSE_DATA_PATH: ClassVar[ - str - ] = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" + SPDX_LICENSE_DATA_PATH: ClassVar[str] = ( + "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" + ) id: str reference: Optional[HttpUrlString] = None From f0578b49787b6074ed4f0fb83f810e240603d269 Mon Sep 17 00:00:00 2001 From: cwognum Date: Mon, 18 Mar 2024 10:47:36 -0400 Subject: [PATCH 9/9] Allow converters to be specified in the factory constructor --- polaris/dataset/_factory.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/polaris/dataset/_factory.py b/polaris/dataset/_factory.py index 3e5d16b6..aff4ba93 100644 --- a/polaris/dataset/_factory.py +++ b/polaris/dataset/_factory.py @@ -53,15 +53,23 @@ class DatasetFactory: the Python API? """ - def __init__(self, zarr_root_path: Optional[str] = None) -> None: + def __init__( + self, zarr_root_path: Optional[str] = None, converters: Optional[Dict[str, Converter]] = None + ) -> None: """ Create a new factory object. Args: zarr_root_path: The root path of the zarr hierarchy. If you want to use pointer columns, this arguments needs to be passed. + converters: The converters to use for specific file types. + You can also register them later with register_converter(). """ - self._converters: Dict[str, Converter] = {} + + if converters is None: + converters = {} + + self._converters: Dict[str, Converter] = converters self.reset(zarr_root_path=zarr_root_path) @property