-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor name lookup for templates/formatters/composites
Now wherever a config file has keys that can be StorageClass names, DatasetType names or "instrument<xxx>" overrides, the parsing of this is done in one place and a new LookupKey object is created and used as the key in the mappings. This provides the ground work for supporting Dimensions in config files.
- Loading branch information
Showing
7 changed files
with
258 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| # This file is part of daf_butler. | ||
| # | ||
| # Developed for the LSST Data Management System. | ||
| # This product includes software developed by the LSST Project | ||
| # (http://www.lsst.org). | ||
| # See the COPYRIGHT file at the top-level directory of this distribution | ||
| # for details of code ownership. | ||
| # | ||
| # This program is free software: you can redistribute it and/or modify | ||
| # it under the terms of the GNU General Public License as published by | ||
| # the Free Software Foundation, either version 3 of the License, or | ||
| # (at your option) any later version. | ||
| # | ||
| # This program is distributed in the hope that it will be useful, | ||
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| # GNU General Public License for more details. | ||
| # | ||
| # You should have received a copy of the GNU General Public License | ||
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| """Support for configuration snippets""" | ||
|
|
||
| __all__ = ("LookupKey", "processLookupConfigs") | ||
|
|
||
| import logging | ||
| import re | ||
| from collections.abc import Mapping | ||
| from .dimensions import DimensionNameSet | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
|
|
||
| class LookupKey: | ||
| """Representation of key that can be used to lookup information based | ||
| on dataset type name, storage class name, dimensions. | ||
| Parameters | ||
| ---------- | ||
| name : `str`, optional | ||
| Primary index string for lookup. If this string looks like it | ||
| represents dimensions (via ``dim1+dim2+dim3`` syntax) the name | ||
| is converted to a `DimensionNameSet` and stored in ``dimensions`` | ||
| property. | ||
| dimensions : `DimensionNameSet`, optional | ||
| Dimensions that are relevant for lookup. Should not be specified | ||
| if ``name`` is also specified. | ||
| dataId : `dict`, optional | ||
| Keys and values from a dataId that should control lookups. | ||
| """ | ||
|
|
||
| def __init__(self, name=None, dimensions=None, dataId=None): | ||
| if name is None and dimensions is None: | ||
| raise ValueError("At least one of name or dimensions must be given") | ||
|
|
||
| if name is not None and dimensions is not None: | ||
| raise ValueError("Can only accept one of name or dimensions") | ||
|
|
||
| self._dimensions = None | ||
| self._name = None | ||
|
|
||
| if name is not None: | ||
| if "+" in name: | ||
| self._dimensions = DimensionNameSet(name.split("+")) | ||
| else: | ||
| self._name = name | ||
| else: | ||
| self._dimensions = dimensions | ||
|
|
||
| # The dataId is converted to a frozenset of key/value | ||
| # tuples. | ||
| if dataId is not None: | ||
| self._dataId = frozenset((k, v) for k, v in dataId.items()) | ||
| else: | ||
| self._dataId = None | ||
|
|
||
| def __str__(self): | ||
| return "({}, {})".format(self._name if self._name else self._dimensions, | ||
| ",".join(str(t) for t in self._dataId) if self._dataId else "") | ||
|
|
||
| def __repr__(self): | ||
| params = "" | ||
| if self.name: | ||
| params += f"name={self.name!r}," | ||
| if self.dimensions: | ||
| params += f"dimensions={self.dimensions!r}," | ||
| if self._dataId: | ||
| params += "dataId={" + ",".join(f"'{k}': {v!r}" for k, v in self._dataId) + "}" | ||
|
|
||
| return f"{self.__class__.__name__}({params})" | ||
|
|
||
| def __eq__(self, other): | ||
| if self._name == other._name and self._dimensions == other._dimensions and \ | ||
| self._dataId == other._dataId: | ||
| return True | ||
| return False | ||
|
|
||
| @property | ||
| def name(self): | ||
| """Primary name string to use as lookup.""" | ||
| return self._name | ||
|
|
||
| @property | ||
| def dimensions(self): | ||
| """Dimensions associated with lookup.""" | ||
| return self._dimensions | ||
|
|
||
| @property | ||
| def dataId(self): | ||
| """Set of key/value tuples that are important for dataId lookup.""" | ||
| return self._dataId | ||
|
|
||
| def __hash__(self): | ||
| """Hash the lookup to allow use as a key in a dict.""" | ||
| return hash((self._name, self._dimensions, self._dataId)) | ||
|
|
||
| def clone(self, name=None, dimensions=None, dataId=None): | ||
| """Clone the object, overriding some options. | ||
| Used to create a new instance of the object whilst updating | ||
| some of it. | ||
| Parameters | ||
| ---------- | ||
| name : `str`, optional | ||
| Primary index string for lookup. Will override ``dimensions`` | ||
| if ``dimensions`` are set. | ||
| dimensions : `DimensionNameSet`, optional | ||
| Dimensions that are relevant for lookup. Will override ``name`` | ||
| if ``name`` is already set. | ||
| dataId : `dict`, optional | ||
| Keys and values from a dataId that should control lookups. | ||
| Returns | ||
| ------- | ||
| clone : `LookupKey` | ||
| Copy with updates. | ||
| """ | ||
| if name is not None and dimensions is not None: | ||
| raise ValueError("Both name and dimensions can not be set") | ||
|
|
||
| # if neither name nor dimensions are specified we copy from current | ||
| # object. Otherwise we'll use the supplied values | ||
| if name is None and dimensions is None: | ||
| name = self._name | ||
| dimensions = self._dimensions | ||
|
|
||
| # To copy the dataId we need to convert it back to a dict when | ||
| # copying | ||
| if dataId is None and self._dataId is not None: | ||
| dataId = {k: v for k, v in self._dataId} | ||
|
|
||
| return self.__class__(name=name, dimensions=dimensions, dataId=dataId) | ||
|
|
||
|
|
||
| def processLookupConfigs(config): | ||
| """Process sections of configuration relating to lookups by dataset type | ||
| name, storage class name, dataId components or dimensions. | ||
| Parameters | ||
| ---------- | ||
| config : `Config` | ||
| A `Config` representing a configuration mapping keys to values where | ||
| the keys can be dataset type names, storage class names, dimensions | ||
| or dataId components. | ||
| Returns | ||
| ------- | ||
| contents : `dict` of `LookupKey` to `str` | ||
| A `dict` with keys constructed from the configuration keys and values | ||
| being simple strings. It is assumed the caller will convert the | ||
| values to the required form. | ||
| """ | ||
| contents = {} | ||
| for name, value in config.items(): | ||
| if isinstance(value, Mapping): | ||
| # indicates a dataId component -- check the format | ||
| kv = re.match(r"([a-z_]+)<(.*)>$", name) | ||
| if kv: | ||
| dataIdKey = kv.group(1) | ||
| dataIdValue = kv.group(2) | ||
| for subKey, subStr in value.items(): | ||
| lookup = LookupKey(name=subKey, dataId={dataIdKey: dataIdValue}) | ||
| contents[lookup] = subStr | ||
| else: | ||
| log.warning("Hierarchical key '%s' not in form 'key<value>'", name) | ||
| else: | ||
| lookup = LookupKey(name=name) | ||
| contents[lookup] = value | ||
|
|
||
| for k, v in contents.items(): | ||
| print(f"{k}: {v}") | ||
| return contents |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.