diff --git a/imagine/observables/dataset.py b/imagine/observables/dataset.py index 06b969d1..7cdb7fe4 100644 --- a/imagine/observables/dataset.py +++ b/imagine/observables/dataset.py @@ -33,6 +33,7 @@ def __init__(self): self._cov = None self._error = None self._data = None + self.otype = None @property @req_attr @@ -174,6 +175,7 @@ def __init__(self, data, name, *, data_col=None, units=None, # Set Nside self.Nside = "tab" + self.otype = "tabular" class HEALPixDataset(Dataset): @@ -194,6 +196,8 @@ def __init__(self, data, error=None, cov=None, Nside=None): raise self.Nside = Nside + self.otype = 'HEALPix' + assert len(data.shape) == 1 self._data = data diff --git a/imagine/observables/observable.py b/imagine/observables/observable.py index 195ba801..b7602142 100644 --- a/imagine/observables/observable.py +++ b/imagine/observables/observable.py @@ -52,9 +52,12 @@ class Observable(object): distributed/copied data dtype : str Data type, must be either: 'measured', 'simulated' or 'covariance' + otype : str + Observable type, must be either: 'HEALPix', 'tabular' or 'plain' """ - def __init__(self, data=None, dtype=None, coords=None): + def __init__(self, data=None, dtype=None, coords=None, otype=None): self.dtype = dtype + self.otype = otype if isinstance(data, u.Quantity): self.data = data.value @@ -75,6 +78,24 @@ def data(self): """ return self._data + @data.setter + def data(self, data): + """ + extra input format check for 'measured' and 'covariance' + no extra check for 'simulated' + """ + log.debug('@ observable::data') + if data is None: + self._data = None + else: + assert (len(data.shape) == 2) + assert isinstance(data, np.ndarray) + if (self._dtype == 'measured'): # copy single-row data from memory + assert (data.shape[0] == 1) + self._data = np.copy(data) + if (self._dtype == 'covariance'): + assert np.equal(*self.shape) + @property def shape(self): """ @@ -112,13 +133,6 @@ def ensemble_mean(self): else: raise TypeError('unsupported data type') - @property - def rw_flag(self): - """ - Rewriting flag, if true, append method will perform rewriting - """ - return self._rw_flag - @property def dtype(self): """ @@ -126,32 +140,21 @@ def dtype(self): """ return self._dtype - @data.setter - def data(self, data): - """ - extra input format check for 'measured' and 'covariance' - no extra check for 'simulated' - """ - log.debug('@ observable::data') - if data is None: - self._data = None - else: - assert (len(data.shape) == 2) - assert isinstance(data, np.ndarray) - if (self._dtype == 'measured'): # copy single-row data from memory - assert (data.shape[0] == 1) - self._data = np.copy(data) - if (self._dtype == 'covariance'): - assert np.equal(*self.shape) - @dtype.setter def dtype(self, dtype): if dtype is None: - raise ValueError('dtype cannot be none') + raise ValueError('dtype cannot be None') else: assert (dtype in ('measured', 'simulated', 'covariance')) self._dtype = str(dtype) + @property + def rw_flag(self): + """ + Rewriting flag, if true, append method will perform rewriting + """ + return self._rw_flag + @rw_flag.setter def rw_flag(self, rw_flag): assert isinstance(rw_flag, bool) diff --git a/imagine/observables/observable_dict.py b/imagine/observables/observable_dict.py index f0e4da41..4a517709 100644 --- a/imagine/observables/observable_dict.py +++ b/imagine/observables/observable_dict.py @@ -109,7 +109,7 @@ def __getitem__(self, key): return self._archive[key] @abc.abstractmethod - def append(self, dataset=None, *, name=None, data=None, plain=False, + def append(self, dataset=None, *, name=None, data=None, otype=None, coords=None): """ Adds/updates name and data @@ -126,9 +126,11 @@ def append(self, dataset=None, *, name=None, data=None, plain=False, `ext` can be 'I','Q','U','PI','PA', None or other customized tags. data : numpy.ndarray or imagine.observables.observable.Observable distributed/copied :py:class:`numpy.ndarray` or :py:class:`Observable ` - plain : bool - If True, means unstructured/tabular data. - If False (default case), means HEALPix-like sky map. + otype : str + Type of observable. May be: 'HEALPix', for HEALPix-like sky map; + 'tabular', for tabular data; or 'plain' for unstructured data. + coords : dict + A dictionary containing the coordinates of tabular data """ if dataset is not None: @@ -137,15 +139,12 @@ def append(self, dataset=None, *, name=None, data=None, plain=False, data = dataset.data cov = dataset.cov coords = dataset.coords - if isinstance(dataset, HEALPixDataset): - plain=False - else: - plain=True + otype = dataset.otype else: cov = data assert (len(name) == 4), 'Wrong format for Observable key!' - return(name, data, cov, plain, coords) + return(name, data, cov, otype, coords) @@ -180,16 +179,16 @@ class Masks(ObservableDict): def append(self, *args, **kwargs): log.debug('@ observable_dict::Masks::append') - name, data, _, plain, _ = super().append(*args, **kwargs) + name, data, _, otype, _ = super().append(*args, **kwargs) if isinstance(data, Observable): assert (data.dtype == 'measured') - if not plain: + if otype == 'HEALPix': assert (data.size == 12*np.uint(name[2])**2) self._archive.update({name: data}) elif isinstance(data, np.ndarray): assert (data.shape[0] == 1) - if not plain: + if otype == 'HEALPix': assert (data.shape[1] == _Nside_to_Npixels(name[2])) self._archive.update({name: Observable(data, 'measured')}) else: @@ -237,9 +236,10 @@ def __call__(self, observable_dict): new_name = (name[0], name[1], masked_data.shape[1], name[3]) masked_dict.append(name=new_name, data=masked_data, - plain=True) + otype='plain') masked_dict.coords = observable.coords masked_dict.unit = observable.unit + return masked_dict @@ -253,17 +253,18 @@ class Measurements(ObservableDict): def append(self, *args, **kwargs): log.debug('@ observable_dict::Measurements::append') - name, data, _, plain, coords = super().append(*args, **kwargs) + name, data, _, otype, coords = super().append(*args, **kwargs) if isinstance(data, Observable): assert (data.dtype == 'measured') self._archive.update({name: data}) elif isinstance(data, np.ndarray): - if not plain: + if otype == 'HEALPix': assert (data.shape[1] == _Nside_to_Npixels(name[2])) self._archive.update({name: Observable(data=data, dtype='measured', - coords=coords)}) + coords=coords, + otype=otype)}) else: raise TypeError('Unsupported data type') @@ -278,7 +279,7 @@ class Simulations(ObservableDict): def append(self, *args, **kwargs): log.debug('@ observable_dict::Simulations::append') - name, data, *_, coords = super().append(*args, **kwargs) + name, data, _, otype, coords = super().append(*args, **kwargs) if name in self._archive.keys(): # app self._archive[name].rw_flag = False @@ -289,7 +290,8 @@ def append(self, *args, **kwargs): elif isinstance(data, np.ndarray): # distributed data self._archive.update({name: Observable(data=data, dtype='simulated', - coords=coords)}) + coords=coords, + otype=otype)}) else: raise TypeError('unsupported data type') @@ -304,12 +306,12 @@ class Covariances(ObservableDict): def append(self, *args, **kwargs): log.debug('@ observable_dict::Covariances::append') - name, _, data, plain, _ = super().append(*args, **kwargs) + name, _, data, otype, _ = super().append(*args, **kwargs) if isinstance(data, Observable): # always rewrite self._archive.update({name: data}) # rw elif isinstance(data, np.ndarray): - if not plain: + if otype == 'HEALPix': assert (data.shape[1] == _Nside_to_Npixels(name[2])) self._archive.update({name: Observable(data, 'covariance')}) else: diff --git a/imagine/simulators/simulator.py b/imagine/simulators/simulator.py index 1c01ef08..1044574e 100644 --- a/imagine/simulators/simulator.py +++ b/imagine/simulators/simulator.py @@ -57,6 +57,7 @@ def __init__(self, measurements): self.observables = [] self.output_coords = {} self.output_units = {} + self.output_type = {} self._ensemble_size = None self.register_observables(measurements) @@ -79,6 +80,7 @@ def register_observables(self, measurements): # perhaps, be useful later self.observables.append(key) self.output_coords[key] = measurements[key].coords + self.output_type[key] = measurements[key].otype self.output_units[key] = measurements[key].unit assert len(self.observables) > 0, 'No valid observable was requested!' @@ -415,5 +417,6 @@ def __call__(self, field_list): output_units=self.output_units[key]) sims.append(name=key, data=sim[np.newaxis, :].to(self.output_units[key]), - coords=self.output_coords[key]) + coords=self.output_coords[key], + otype=self.output_type[key]) return sims diff --git a/imagine/tests/test_likelihood.py b/imagine/tests/test_likelihood.py index 3a20c5aa..851be689 100644 --- a/imagine/tests/test_likelihood.py +++ b/imagine/tests/test_likelihood.py @@ -26,11 +26,11 @@ def test_without_cov(self): arr_a = np.random.rand(1, 48) comm.Bcast(arr_a, root=0) mea = Observable(arr_a, 'measured') - meadict.append(name=('test', None, 2, None), data=mea) + meadict.append(name=('test', None, 2, None), data=mea, otype='HEALPix') # mock sims arr_b = np.random.rand(3, 48) sim = Observable(arr_b, 'simulated') - simdict.append(name=('test', None, 2, None), data=sim) + simdict.append(name=('test', None, 2, None), data=sim, otype='HEALPix') # no covariance lh = SimpleLikelihood(meadict) # calc by likelihood @@ -51,19 +51,17 @@ def test_with_cov(self): comm.Bcast(arr_a, root=0) mea = Observable(arr_a, 'measured') meadict.append(name=('test', None, 4*mpisize, None), - data=mea, - plain=True) + data=mea, otype='plain') # mock sims arr_b = np.random.rand(5, 4*mpisize) sim = Observable(arr_b, 'simulated') simdict.append(name=('test', None, 4*mpisize, None), - data=sim) + data=sim, otype='plain') # mock covariance arr_c = np.random.rand(4, 4*mpisize) cov = Observable(arr_c, 'covariance') covdict.append(name=('test', None, 4*mpisize, None), - data=cov, - plain=True) + data=cov) # with covariance lh = SimpleLikelihood(meadict, covdict) # calc by likelihood @@ -87,14 +85,12 @@ def test_without_simcov(self): comm.Bcast(arr_a, root=0) mea = Observable(arr_a, 'measured') meadict.append(name=('test', None, 4*mpisize, None), - data=mea, - plain=True) + data=mea, otype='plain') # mock covariance arr_c = np.random.rand(4, 4*mpisize) cov = Observable(arr_c, 'covariance') covdict.append(name=('test', None, 4*mpisize, None), - data=cov, - plain=True) + data=cov) # mock observable with repeated single realisation arr_b = np.random.rand(1, 4*mpisize) comm.Bcast(arr_b, root=0) @@ -103,7 +99,7 @@ def test_without_simcov(self): arr_ens[i] = arr_b sim = Observable(arr_ens, 'simulated') simdict.append(name=('test', None, 4*mpisize, None), - data=sim) + data=sim, otype='plain') # simplelikelihood lh_simple = SimpleLikelihood(meadict, covdict) rslt_simple = lh_simple(simdict) @@ -120,7 +116,7 @@ def test_without_cov(self): comm.Bcast(arr_a, root=0) mea = Observable(arr_a, 'measured') meadict.append(name=('test', None, mpisize, None), - data=mea) + data=mea, otype='HEALPix') # mock observable with repeated single realisation arr_b = np.random.rand(1, 12*mpisize**2) comm.Bcast(arr_b, root=0) @@ -132,7 +128,7 @@ def test_without_cov(self): arr_ens[i] = arr_b sim = Observable(arr_ens, 'simulated') simdict.append(name=('test', None, mpisize, None), - data=sim) + data=sim, otype='HEALPix') # simplelikelihood lh_simple = SimpleLikelihood(meadict) rslt_simple = lh_simple(simdict) diff --git a/imagine/tests/test_observabledict.py b/imagine/tests/test_observabledict.py index ca592408..f89a00bc 100644 --- a/imagine/tests/test_observabledict.py +++ b/imagine/tests/test_observabledict.py @@ -27,13 +27,14 @@ def test_measuredict_append_array(self): measuredict = Measurements() measuredict.append(name=('test', None, 3, None), data=arr, - plain=True) # plain array + otype='plain') # plain array local_arr = measuredict[('test', None, 3, None)].data if mpirank == 0: assert np.allclose(local_arr[0], arr[0]) hrr = np.random.rand(1, 48) measuredict.append(name=('test', None, 2, None), - data=hrr) # healpix array + data=hrr, + otype='HEALPix') # healpix array local_arr = measuredict[('test', None, 2, None)].data if mpirank == 0: assert np.allclose(local_arr[0], hrr[0]) @@ -49,7 +50,7 @@ def test_measuredict_append_observable(self): obs2 = Observable(arr, 'measured') measuredict.append(name=('test', None, 3, None), data=obs2, - plain=True) # plain Observable + otype='plain') # plain Observable assert np.allclose(measuredict[('test', None, 3, None)].data[0], arr[0]) def test_simdict_append_array(self): @@ -57,12 +58,13 @@ def test_simdict_append_array(self): simdict = Simulations() simdict.append(name=('test', None, 3, None), data=arr, - plain=True) # plain array + otype='plain') # plain array assert simdict[('test', None, 3, None)].shape == (2*mpisize, 3) assert np.allclose(simdict[('test', None, 3, None)].data, arr) hrr = np.random.rand(3, 48) simdict.append(name=('test', None, 2, None), - data=hrr) # healpix array + data=hrr, + otype='HEALPix') # healpix array assert simdict[('test', None, 2, None)].shape == (3*mpisize, 48) assert np.allclose(simdict[('test', None, 2, None)].data, hrr) @@ -71,11 +73,11 @@ def test_simdict_append_array_twice(self): simdict = Simulations() simdict.append(name=('test', None, 3, None), data=arr, - plain=True) # plain array + otype='plain') # plain array assert simdict[('test', None, 3, None)].shape == (2*mpisize, 3) simdict.append(name=('test', None, 3, None), data=arr, - plain=True) # plain array + otype='plain') # plain array assert simdict[('test', None, 3, None)].shape == (4*mpisize, 3) def test_simdict_append_observable(self): @@ -83,14 +85,15 @@ def test_simdict_append_observable(self): obs1 = Observable(hrr, 'simulated') simdict = Simulations() simdict.append(name=('test', None, 2, None), - data=obs1) # healpix Observable + data=obs1, + otype='HEALPix') # healpix Observable assert simdict[('test', None, 2, None)].shape == (2*mpisize, 48) assert np.allclose(simdict[('test', None, 2, None)].data, hrr) arr = np.random.rand(5, 3) obs2 = Observable(arr, 'simulated') simdict.append(name=('test', None, 3, None), data=obs2, - plain=True) # plain Observable + otype='plain') # plain Observable assert simdict[('test', None, 3, None)].shape == (5*mpisize, 3) assert np.allclose(simdict[('test', None, 3, None)].data, arr) @@ -98,8 +101,7 @@ def test_covdict_append_array(self): cov = np.random.rand(2, 2*mpisize) covdict = Covariances() covdict.append(name=('test', None, 2*mpisize, None), - data=cov, - plain=True) # plain covariance + data=cov) # plain covariance assert covdict[('test', None, 2*mpisize, None)].shape == (2*mpisize, 2*mpisize) assert np.allclose(covdict[('test', None, 2*mpisize, None)].data, cov) cov = np.random.rand(12*mpisize, 12*mpisize*mpisize) @@ -112,8 +114,7 @@ def test_covdict_append_observable(self): cov = Observable(np.random.rand(2, 2*mpisize), 'covariance') covdict = Covariances() covdict.append(name=('test', None, 2*mpisize, None), - data=cov, - plain=True) # plain covariance + data=cov) # plain covariance assert np.allclose(covdict[('test', None, 2*mpisize, None)].data, cov.data) cov = Observable(np.random.rand(12*mpisize, 12*mpisize*mpisize), 'covariance') covdict.append(name=('test', None, mpisize, None), @@ -129,8 +130,7 @@ def test_maskdict_append_array(self): local_msk = mskdict[('test', None, 2, None)].data assert np.allclose(local_msk[0], msk[0]) mskdict.append(name=('test', None, 48, None), - data=msk, - plain=True) + data=msk) local_msk = mskdict[('test', None, 48, None)].data assert np.allclose(local_msk[0], msk[0]) @@ -139,13 +139,12 @@ def test_meadict_apply_mask(self): mskdict = Masks() comm.Bcast(msk, root=0) mskdict.append(name=('test', None, 5, None), - data=msk, - plain=True) + data=msk) arr = np.array([0., 1., 2., 3., 4.]).reshape(1, 5) meadict = Measurements() meadict.append(name=('test', None, 5, None), data=arr, - plain=True) + otype='plain') meadict = mskdict(meadict) assert np.allclose(meadict[('test', None, 3, None)].data[0], [1, 3, 4]) # HEALPix map @@ -155,7 +154,7 @@ def test_meadict_apply_mask(self): data=msk) arr = np.random.rand(1, 48) meadict.append(name=('test', None, 2, None), - data=arr) + data=arr, otype='HEALPix') pix_num = msk.sum() meadict = mskdict(meadict) assert ('test', None, pix_num, None) in meadict.keys() @@ -165,13 +164,11 @@ def test_covdict_apply_mask(self): mskdict = Masks() comm.Bcast(msk, root=0) mskdict.append(name=('test', None, 2*mpisize, None), - data=msk, - plain=True) + data=msk, otype='plain') cov = np.random.rand(2, 2*mpisize) covdict = Covariances() covdict.append(name=('test', None, 2*mpisize, None), - data=cov, - plain=True) + data=cov) covdict = mskdict(covdict) pix_num = msk.sum() assert ('test', None, pix_num, None) in covdict.keys()