From a92a0faa50b115a833933f849d8e3ceca88974db Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 16 Aug 2023 09:21:33 -0700 Subject: [PATCH 01/80] Temp commit --- .../generate_rich_mean_mass_sacc_data.py | 0 firecrown/modeling_tools.py | 6 ++ firecrown/models/cluster_theory/__init__.py | 0 .../cluster_theory/cluster_abundance.py | 73 +++++++++++++++++++ 4 files changed, 79 insertions(+) mode change 100644 => 100755 examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py create mode 100644 firecrown/models/cluster_theory/__init__.py create mode 100644 firecrown/models/cluster_theory/cluster_abundance.py diff --git a/examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py b/examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py old mode 100644 new mode 100755 diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index e901264e6..b9bb951ca 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -7,6 +7,7 @@ from typing import Dict, Optional, final import pyccl.nl_pt +from .models.cluster_theory import ClusterAbundance class ModelingTools: @@ -17,10 +18,12 @@ def __init__( self, *, pt_calculator: Optional[pyccl.nl_pt.EulerianPTCalculator] = None, + cluster_abundance: Optional[ClusterAbundance] = None, ): self.ccl_cosmo: Optional[pyccl.Cosmology] = None self.pt_calculator: Optional[pyccl.nl_pt.EulerianPTCalculator] = pt_calculator self.powerspectra: Dict[str, pyccl.Pk2D] = {} + self.cluster_abundance = cluster_abundance def add_pk(self, name: str, powerspectrum: pyccl.Pk2D): """Add a :python:`pyccl.Pk2D` to the table of power spectra.""" @@ -70,6 +73,9 @@ def prepare(self, ccl_cosmo: pyccl.Cosmology) -> None: if self.pt_calculator is not None: self.pt_calculator.update_ingredients(ccl_cosmo) + if self.cluster_abundance is not None: + self.cluster_abundance.update_ingredients(ccl_cosmo) + @final def reset(self) -> None: """Resets all CCL objects in ModelingTools.""" diff --git a/firecrown/models/cluster_theory/__init__.py b/firecrown/models/cluster_theory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/firecrown/models/cluster_theory/cluster_abundance.py b/firecrown/models/cluster_theory/cluster_abundance.py new file mode 100644 index 000000000..950fb22ac --- /dev/null +++ b/firecrown/models/cluster_theory/cluster_abundance.py @@ -0,0 +1,73 @@ +from pyccl.cosmology import Cosmology +import pyccl.background as bkg +import pyccl + + +class ClusterAbundance(object): + def __init__( + self, + cosmo: Cosmology, + halo_mass_function: pyccl.halos.MassFunc, + sky_area: float, + ): + self.cosmo = cosmo + self.halo_mass_function = halo_mass_function + self.sky_area = sky_area + + def update_ingredients(self, cosmo: Cosmology): + self.cosmo = cosmo + # Set cosmology dependent cluster abundance values. + pass + + def comoving_volume(self, z) -> float: + """Differential Comoving Volume at z. + + parameters + :param ccl_cosmo: pyccl Cosmology + :param z: Cluster Redshift. + + :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). + """ + scale_factor = 1.0 / (1.0 + z) + da = bkg.angular_diameter_distance(self.cosmo, scale_factor) + + h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) + dV = ( + ((1.0 + z) ** 2) + * (da**2) + * pyccl.physical_constants.CLIGHT_HMPC + / self.cosmo["h"] + / h_over_h0 + ) + return dV * self.sky_area_rad + + def mf_d2N_dz_dlnM_completeness(self, logM: float, z: float) -> float: + """ + Computes the mass function at z and logM. + + :param ccl_cosmo: pyccl Cosmology + :param logM: Cluster mass given by log10(M) where M is in units of + M_sun (comoving). + :param z: Cluster Redshift. + :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). + """ + mass_function = self.mass_funcion(self.cosmo, logM, z) + comoving_volume = self.comoving_volume(self.cosmo, z) + completeness = self._cluster_abundance_compute_completeness(logM, z) + + return mass_function * comoving_volume * completeness + + def mass_funcion(self, logM: float, z: float) -> float: + """ + Computes the mass function at z and logM. + + :param ccl_cosmo: pyccl Cosmology + :param logM: Cluster mass given by log10(M) where M is in units of + M_sun (comoving). + :param z: Cluster Redshift. + :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). + """ + scale_factor = 1.0 / (1.0 + z) + mass = 10 ** (logM) + nm = self.halo_mass_function(self.cosmo, mass, scale_factor) + return nm From 731c5673ce20c26ef60ccc93ef2d9edab1f278eb Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 5 Oct 2023 10:54:33 -0700 Subject: [PATCH 02/80] Midstage of refactor. Removed old cluster number count classes Added new Kernel base class, with MassObservable and Redshift subclasses Added some simple implementations of Kernel Added Kernel Factories Still WIP. --- .../cluster_redshift_richness.py | 54 ++- .../compare_integration_methods.py | 2 +- .../statistic/cluster_number_counts.py | 180 ++------- firecrown/models/cluster_abundance.py | 362 ++---------------- firecrown/models/cluster_mass.py | 86 ----- firecrown/models/cluster_mass_rich_proxy.py | 215 ----------- firecrown/models/cluster_mass_true.py | 75 ---- firecrown/models/cluster_redshift.py | 87 ----- firecrown/models/cluster_redshift_spec.py | 73 ---- firecrown/models/cluster_theory/__init__.py | 0 .../cluster_theory/cluster_abundance.py | 73 ---- firecrown/models/kernel.py | 49 +++ firecrown/models/kernel_factory.py | 21 + firecrown/models/mass_observable.py | 100 +++++ firecrown/models/mass_observable_factory.py | 29 ++ firecrown/models/redshift.py | 32 ++ firecrown/models/redshift_factory.py | 19 + firecrown/models/sacc_adapter.py | 69 ++++ tests/test_cluster.py | 2 +- 19 files changed, 417 insertions(+), 1111 deletions(-) delete mode 100644 firecrown/models/cluster_mass.py delete mode 100644 firecrown/models/cluster_mass_rich_proxy.py delete mode 100644 firecrown/models/cluster_mass_true.py delete mode 100644 firecrown/models/cluster_redshift.py delete mode 100644 firecrown/models/cluster_redshift_spec.py delete mode 100644 firecrown/models/cluster_theory/__init__.py delete mode 100644 firecrown/models/cluster_theory/cluster_abundance.py create mode 100644 firecrown/models/kernel.py create mode 100644 firecrown/models/kernel_factory.py create mode 100644 firecrown/models/mass_observable.py create mode 100644 firecrown/models/mass_observable_factory.py create mode 100644 firecrown/models/redshift.py create mode 100644 firecrown/models/redshift_factory.py create mode 100644 firecrown/models/sacc_adapter.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 0077050ef..3f10a33d5 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -12,8 +12,9 @@ from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster_abundance import ClusterAbundance -from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich -from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec +from firecrown.models.mass_observable import MassObservableFactory, MassObservableType +from firecrown.models.redshift import RedshiftFactory, RedshiftType +from firecrown.parameters import ParamsMap def build_likelihood(build_parameters): @@ -21,31 +22,9 @@ def build_likelihood(build_parameters): Here we instantiate the number density (or mass function) object. """ - pivot_mass = 14.625862906 - pivot_redshift = 0.6 - - cluster_mass_r = ClusterMassRich(pivot_mass, pivot_redshift) - cluster_z = ClusterRedshiftSpec() - - # TODO: remove try/except when pyccl 3.0 is released - try: - hmd_200 = ccl.halos.MassDef200m() - except TypeError: - hmd_200 = ccl.halos.MassDef200m - - hmf_args: Dict[str, Any] = {} - hmf_name = "Tinker08" - - cluster_abundance = ClusterAbundance(hmd_200, hmf_name, hmf_args) - - stats = ClusterNumberCounts( - "numcosmo_simulated_redshift_richness", - cluster_abundance, - cluster_mass_r, - cluster_z, - use_cluster_counts=build_parameters.get_bool("use_cluster_counts", True), - use_mean_log_mass=build_parameters.get_bool("use_mean_log_mass", False), - ) + stats = ClusterNumberCounts(survey_name="numcosmo_simulated_redshift_richness") + stats.use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) + stats.use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) stats_list = [stats] # Here we instantiate the actual likelihood. The statistics argument carry @@ -70,6 +49,25 @@ def build_likelihood(build_parameters): # This script will be loaded by the appropriated connector. The framework # then looks for the `likelihood` variable to find the instance that will # be used to compute the likelihood. - modeling = ModelingTools() + hmf = ccl.halos.MassFuncTinker08() + cluster_abundance = ClusterAbundance(hmf) + + mass_observable = MassObservableFactory.create( + MassObservableType.MU_SIGMA, + ParamsMap( + { + "pivot_mass": 14.625862906, + "pivot_redshift": 0.6, + "min_mass": 13.0, + "max_mass": 15.0, + } + ), + ) + cluster_abundance.add_kernel(mass_observable) + + redshift = RedshiftFactory.create(RedshiftType.SPEC) + cluster_abundance.add_kernel(redshift) + + modeling = ModelingTools(cluster_abundance=cluster_abundance) return lk, modeling diff --git a/examples/cluster_number_counts/compare_integration_methods.py b/examples/cluster_number_counts/compare_integration_methods.py index 62bc92adf..f6b135dbb 100644 --- a/examples/cluster_number_counts/compare_integration_methods.py +++ b/examples/cluster_number_counts/compare_integration_methods.py @@ -8,7 +8,7 @@ import numpy as np from numcosmo_py import Ncm -from firecrown.models.cluster_abundance import ClusterAbundance +from firecrown.models.cluster_abundance_old import ClusterAbundance from firecrown.models.cluster_mass_rich_proxy import ( ClusterMassRich, ClusterMassRichBinArgument, diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index a7d41b3fc..d23ff8cba 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -1,9 +1,3 @@ -"""Cluster Number Count statistic support. -This module reads the necessary data from a SACC file to compute the -theoretical prediction of cluster number counts inside bins of redshift -and a mass proxy. -""" - from __future__ import annotations from typing import List, Dict, Tuple, Optional @@ -15,182 +9,78 @@ from .statistic import Statistic, DataVector, TheoryVector from .source.source import SourceSystematic from ....models.cluster_abundance import ClusterAbundance -from ....models.cluster_mass import ClusterMass, ClusterMassArgument -from ....models.cluster_redshift import ClusterRedshift, ClusterRedshiftArgument from ....modeling_tools import ModelingTools +from ....models.sacc_adapter import SaccAdapter class ClusterNumberCounts(Statistic): - """A Cluster Number Count statistic (e.g., halo mass function, - multiplicity functions, volume element, etc.). - This subclass implements the read and computes method for - the Statistic class. It is used to compute the theoretical prediction of - cluster number counts. - """ + @property + def use_cluster_counts(self) -> bool: + return self._use_cluster_counts + + @property + def use_mean_log_mass(self) -> bool: + return self._use_mean_log_mass + + @use_cluster_counts.setter + def use_cluster_counts(self, value: bool): + self._use_cluster_counts = value + + @use_mean_log_mass.setter + def use_mean_log_mass(self, value: bool): + self._use_mean_log_mass = value def __init__( self, - survey_tracer: str, - cluster_abundance: ClusterAbundance, - cluster_mass: ClusterMass, - cluster_redshift: ClusterRedshift, systematics: Optional[List[SourceSystematic]] = None, - use_cluster_counts: bool = True, - use_mean_log_mass: bool = False, + survey_nm: str = "numcosmo_simulated_redshift_richness", ): - """Initialize the ClusterNumberCounts object. - Parameters - - :param survey_tracer: name of the survey tracer in the SACC data. - :param cluster_abundance: The cluster abundance model to use. - :param systematics: A list of the statistics-level systematics to apply to - the statistic. The default of `None` implies no systematics. - """ - super().__init__(parameter_prefix=survey_tracer) - self.survey_tracer = survey_tracer + super().__init__() self.systematics = systematics or [] - self.data_vector: Optional[DataVector] = None self.theory_vector: Optional[TheoryVector] = None - self.cluster_abundance: ClusterAbundance = cluster_abundance - self.cluster_mass: ClusterMass = cluster_mass - self.cluster_redshift: ClusterRedshift = cluster_redshift - self.tracer_args: List[Tuple[ClusterRedshiftArgument, ClusterMassArgument]] = [] - self.use_cluster_counts: bool = use_cluster_counts - self.use_mean_log_mass: bool = use_mean_log_mass - - if not self.use_cluster_counts and not self.use_mean_log_mass: - raise ValueError( - "At least one of use_cluster_counts and use_mean_log_mass must be True." - ) - - def _read_data_type(self, sacc_data, data_type): - """Internal function to read the data from the SACC file.""" - tracers_combinations = np.array( - sacc_data.get_tracer_combinations(data_type=data_type) - ) - - if len(tracers_combinations) == 0: - raise ValueError( - f"The SACC file does not contain any tracers for the " - f"{data_type} data type." - ) - - if tracers_combinations.shape[1] != 3: - raise ValueError( - "The SACC file must contain 3 tracers for the " - "cluster_counts data type: cluster_survey, " - "redshift argument and mass argument tracers." - ) - - cluster_survey_tracers = tracers_combinations[:, 0] - - if self.survey_tracer not in cluster_survey_tracers: - raise ValueError( - f"The SACC tracer {self.survey_tracer} is not " - f"present in the SACC file." - ) - - survey_selection = cluster_survey_tracers == self.survey_tracer - - z_tracers = np.unique(tracers_combinations[survey_selection, 1]) - logM_tracers = np.unique(tracers_combinations[survey_selection, 2]) - - z_tracer_bins: Dict[str, ClusterRedshiftArgument] = { - z_tracer: self.cluster_redshift.gen_bin_from_tracer( - sacc_data.get_tracer(z_tracer) - ) - for z_tracer in z_tracers - } - logM_tracer_bins: Dict[str, ClusterMassArgument] = { - logM_tracer: self.cluster_mass.gen_bin_from_tracer( - sacc_data.get_tracer(logM_tracer) - ) - for logM_tracer in logM_tracers - } - - self.tracer_args = [ - (z_tracer_bins[z_tracer], logM_tracer_bins[logM_tracer]) - for _, z_tracer, logM_tracer in tracers_combinations[survey_selection] - ] - - self.cluster_abundance.read(sacc_data) - self.cluster_mass.read(sacc_data) - self.cluster_redshift.read(sacc_data) - - data_vector_list = list( - sacc_data.get_mean(data_type=data_type)[survey_selection] - ) - sacc_indices_list = list( - sacc_data.indices(data_type=data_type)[survey_selection] - ) - - return data_vector_list, sacc_indices_list + self.survey_nm = survey_nm def read(self, sacc_data: sacc.Sacc): - """Read the data for this statistic from the SACC file. - - :param sacc_data: The data in the SACC format. - """ - try: - survey_tracer: SurveyTracer = sacc_data.get_tracer(self.survey_tracer) + survey_tracer: SurveyTracer = sacc_data.get_tracer(self.survey_nm) except KeyError as exc: raise ValueError( - f"The SACC file does not contain the SurveyTracer " - f"{self.survey_tracer}." + f"The SACC file does not contain the SurveyTracer " f"{self.survey_nm}." ) from exc if not isinstance(survey_tracer, SurveyTracer): - raise ValueError( - f"The SACC tracer {self.survey_tracer} is not a SurveyTracer." - ) + raise ValueError(f"The SACC tracer {self.survey_nm} is not a SurveyTracer.") - self.cluster_abundance.sky_area = survey_tracer.sky_area + sacc_types = sacc.data_types.standard_types + sa = SaccAdapter(sacc_data) data_vector_list = [] sacc_indices_list = [] if self.use_cluster_counts: - # pylint: disable-next=no-member - cluster_counts = sacc.standard_types.cluster_counts - ( - cluster_counts_data_vector_list, - cluster_counts_sacc_indices_list, - ) = self._read_data_type(sacc_data, cluster_counts) - data_vector_list += cluster_counts_data_vector_list - sacc_indices_list += cluster_counts_sacc_indices_list + dtype = sacc_types.cluster_counts + data, indices = sa.get_data_and_indices(dtype, survey_tracer) + data_vector_list += data + sacc_indices_list += indices if self.use_mean_log_mass: - # pylint: disable-next=no-member - cluster_mean_log_mass = sacc.standard_types.cluster_mean_log_mass - ( - mean_log_mass_data_vector_list, - mean_log_mass_sacc_indices_list, - ) = self._read_data_type(sacc_data, cluster_mean_log_mass) - - data_vector_list += mean_log_mass_data_vector_list - sacc_indices_list += mean_log_mass_sacc_indices_list + dtype = sacc_types.cluster_mean_log_mass + data, indices = sa.get_data_and_indices(dtype, survey_tracer) + data_vector_list += data + sacc_indices_list += indices + self.sky_area = sa.survey_tracer.sky_area self.data_vector = DataVector.from_list(data_vector_list) self.sacc_indices = np.array(sacc_indices_list) super().read(sacc_data) def get_data_vector(self) -> DataVector: - """Return the data vector; raise exception if there is none.""" assert self.data_vector is not None return self.data_vector def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: - """Compute a Number Count statistic using the data from the - Read method, the cosmology object, and the Bocquet16 halo mass function. - - :param tools: ModelingTools firecrown object - used to load the required cosmology. - - :return: Numpy Array of floats - An array with the theoretical prediction of the number of clusters - in each bin of redsfhit and mass. - """ + tools.cluster_abundance.sky_area = self.sky_area ccl_cosmo = tools.get_ccl_cosmology() + # tools.cluster_abundance.etc() theory_vector_list = [] cluster_counts_list = [] diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index 387273085..c0250174a 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -1,115 +1,36 @@ -r"""Cluster Abundance Module -abstract class to compute cluster abundance. -======================================== -The implemented functions use PyCCL library as backend. -""" -from __future__ import annotations -from typing import Optional, Any, Dict, List, Tuple, final +from typing import List +from pyccl.cosmology import Cosmology +import pyccl.background as bkg +import pyccl +from firecrown.models.kernel import Kernel import numpy as np -import pyccl as ccl -from numcosmo_py import Ncm -import scipy.integrate -from ..updatable import Updatable -from .cluster_mass import ClusterMassArgument -from .cluster_redshift import ClusterRedshiftArgument - - -class CountsIntegralND(Ncm.IntegralND): - """Integral subclass used by the ClusterAbundance - to compute the integrals using numcosmo.""" - - def __init__(self, dim, fun, *args): - super().__init__() - self.dim = dim - self.fun = fun - self.args = args - - # pylint: disable-next=arguments-differ - def do_get_dimensions(self) -> Tuple[int, int]: - """Get number of dimensions.""" - return self.dim, 1 - - # pylint: disable-next=arguments-differ - def do_integrand( - self, - x_vec: Ncm.Vector, - dim: int, - npoints: int, - _fdim: int, - fval_vec: Ncm.Vector, - ) -> None: - """Integrand function.""" - x = np.array(x_vec.dup_array()).reshape(npoints, dim) - fval_vec.set_array([self.fun(x_i, *self.args) for x_i in x]) - - -class ClusterAbundance(Updatable): - r"""Cluster Abundance class""" - - def __init__( - self, - halo_mass_definition: ccl.halos.MassDef, - halo_mass_function_name: str, - halo_mass_function_args: Dict[str, Any], - sky_area: float = 100.0, - use_completness: bool = False, - use_purity: bool = False, - integ_method: Ncm.IntegralNDMethod = Ncm.IntegralNDMethod.P_V, - prefer_scipy_integration: bool = False, - reltol: float = 1.0e-4, - abstol: float = 1.0e-12, - ): - """Initialize the ClusterAbundance class. - - :param halo_mass_definition: Halo mass definition. - :param halo_mass_function_name: Halo mass function name. - :param halo_mass_function_args: Halo mass function arguments. - :param sky_area: Sky area in square degrees, defaults to 100 sq deg. - :param use_completness: Use completeness function, defaults to False. - :param use_purity: Use purity function, defaults to False. - - :return: ClusterAbundance object. - """ - super().__init__() - self.sky_area = sky_area - self.halo_mass_definition = halo_mass_definition - self.halo_mass_function_name = halo_mass_function_name - self.halo_mass_function_args = halo_mass_function_args - self.halo_mass_function: Optional[ccl.halos.MassFunc] = None - self.use_purity = use_purity - self.integ_method = integ_method - self.reltol = reltol - self.abstol = abstol - self.prefer_scipy_integration = prefer_scipy_integration - if use_completness: - self.base_mf_d2N_dz_dlnM = self.mf_d2N_dz_dlnM_completeness - else: - self.base_mf_d2N_dz_dlnM = self.mf_d2N_dz_dlnM +class ClusterAbundance(object): @property def sky_area(self) -> float: - """Return the sky area.""" return self.sky_area_rad * (180.0 / np.pi) ** 2 @sky_area.setter def sky_area(self, sky_area: float) -> None: - """Set the sky area.""" self.sky_area_rad = sky_area * (np.pi / 180.0) ** 2 - @final - def _reset(self) -> None: - """Implementation of the Updatable interface method `_reset`.""" - self.halo_mass_function = None + def __init__( + self, + halo_mass_function: pyccl.halos.MassFunc, + ): + self.kernels: List[Kernel] = [] + self.cosmo = None + self.halo_mass_function = halo_mass_function - def read(self, sacc_data): - """Read the data for this statistic from the SACC file. + def add_kernel(self, kernel: Kernel): + self.kernels.append(kernel) - :param sacc_data: The data in the sacc format. - """ + def update_ingredients(self, cosmo: Cosmology): + self.cosmo = cosmo - def dV_dz(self, ccl_cosmo: ccl.Cosmology, z) -> float: + def comoving_volume(self, z) -> float: """Differential Comoving Volume at z. parameters @@ -118,241 +39,28 @@ def dV_dz(self, ccl_cosmo: ccl.Cosmology, z) -> float: :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). """ - a = 1.0 / (1.0 + z) - da = ccl.background.angular_diameter_distance(ccl_cosmo, a) - E = ccl.background.h_over_h0(ccl_cosmo, a) + scale_factor = 1.0 / (1.0 + z) + da = bkg.angular_diameter_distance(self.cosmo, scale_factor) + + h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) dV = ( ((1.0 + z) ** 2) * (da**2) - * ccl.physical_constants.CLIGHT_HMPC - / ccl_cosmo["h"] - / E + * pyccl.physical_constants.CLIGHT_HMPC + / self.cosmo["h"] + / h_over_h0 ) return dV * self.sky_area_rad - def mf_d2N_dV_dlnM(self, ccl_cosmo: ccl.Cosmology, logM: float, z: float) -> float: - """ - Computes the mass function at z and logM. + def mass_function(self, mass: float, z: float) -> float: + scale_factor = 1.0 / (1.0 + z) + hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) + return hmf - :param ccl_cosmo: pyccl Cosmology - :param logM: Cluster mass given by log10(M) where M is in units of - M_sun (comoving). - :param z: Cluster Redshift. - :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). - """ - a = 1.0 / (1.0 + z) - mass = 10 ** (logM) - if self.halo_mass_function is None: - self.halo_mass_function = ccl.halos.MassFunc.from_name( - self.halo_mass_function_name - )(**self.halo_mass_function_args) - nm = self.halo_mass_function(ccl_cosmo, mass, a) - return nm - - def mf_d2N_dz_dlnM(self, ccl_cosmo: ccl.Cosmology, logM: float, z: float) -> float: - """ - Computes the mass function at z and logM. + def build_integrand(self, mass, z): + integrand = self.comoving_volume(z) * self.mass_function(mass, z) + for kernel in self.kernels: + integrand *= kernel.distribution(mass, z) - :param ccl_cosmo: pyccl Cosmology - :param logM: Cluster mass given by log10(M) where M is in units of - M_sun (comoving). - :param z: Cluster Redshift. - :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). - """ - d2N_dV_dlnM = self.mf_d2N_dV_dlnM(ccl_cosmo, logM, z) - dV_dz = self.dV_dz(ccl_cosmo, z) - - return d2N_dV_dlnM * dV_dz - - def mf_d2N_dz_dlnM_completeness( - self, ccl_cosmo: ccl.Cosmology, logM: float, z: float - ) -> float: - """ - Computes the mass function at z and logM. - - :param ccl_cosmo: pyccl Cosmology - :param logM: Cluster mass given by log10(M) where M is in units of - M_sun (comoving). - :param z: Cluster Redshift. - :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). - """ - d2N_dV_dlnM = self.mf_d2N_dV_dlnM(ccl_cosmo, logM, z) - dV_dz = self.dV_dz(ccl_cosmo, z) - completeness = self._cluster_abundance_compute_completeness(logM, z) - - return d2N_dV_dlnM * dV_dz * completeness - - def _cluster_abundance_compute_purity(self, logM_obs, z): - ln_r = np.log(10**logM_obs) - a_nc = np.log(10) * 0.8612 - b_nc = np.log(10) * 0.3527 - a_rc = 2.2183 - b_rc = -0.6592 - nc = a_nc + b_nc * (1.0 + z) - ln_rc = a_rc + b_rc * (1.0 + z) - purity = (ln_r / ln_rc) ** nc / ((ln_r / ln_rc) ** nc + 1.0) - return purity - - def _cluster_abundance_compute_completeness(self, logM, z): - a_nc = 1.1321 - b_nc = 0.7751 - a_mc = 13.31 - b_mc = 0.2025 - log_mc = a_mc + b_mc * (1.0 + z) - nc = a_nc + b_nc * (1.0 + z) - C = (logM / log_mc) ** nc / ((logM / log_mc) ** nc + 1.0) - return C - - def _process_args(self, args): - x = np.array(args[0:-5]) - index_map, arg, ccl_cosmo, mass_arg, redshift_arg = args[-5:] - arg[index_map] = x - redshift_start_index = 2 + redshift_arg.dim - - logM, z = arg[0:2] - proxy_z = arg[2:redshift_start_index] - proxy_m = arg[redshift_start_index:] - - return logM, z, proxy_z, proxy_m, ccl_cosmo, mass_arg, redshift_arg - - # Generic integrand for the cluster abundance - # The arg array always has the following structure: - # [logM, z, proxy_z, proxy_m] - # where proxy_z and proxy_m are the proxy parameters for the redshift and - # mass arguments respectively. - # The index_map array is used to map the proxy parameters to the correct - # position in the arg array. - def _compute_integrand(self, *args): - ( - logM, - z, - proxy_z, - proxy_m, - ccl_cosmo, - mass_arg, - redshift_arg, - ) = self._process_args(args) - - return ( - self.base_mf_d2N_dz_dlnM(ccl_cosmo, logM, z) - * mass_arg.p(logM, z, *proxy_m) - * redshift_arg.p(logM, z, *proxy_z) - ) - - # As above but for the mean mass - def _compute_integrand_mean_logM(self, *args) -> float: - ( - logM, - z, - proxy_z, - proxy_m, - ccl_cosmo, - mass_arg, - redshift_arg, - ) = self._process_args(args) - - return ( - self.base_mf_d2N_dz_dlnM(ccl_cosmo, logM, z) - * mass_arg.p(logM, z, *proxy_m) - * redshift_arg.p(logM, z, *proxy_z) - * logM - ) - - def _compute_any_from_args( - self, - integrand, - ccl_cosmo: ccl.Cosmology, - mass_arg: ClusterMassArgument, - redshift_arg: ClusterRedshiftArgument, - ) -> float: - last_index = 0 - - arg = np.zeros(2 + mass_arg.dim + redshift_arg.dim) - index_map: List[int] = [] - bounds_list: List[Tuple[float, float]] = [] - - if mass_arg.is_dirac_delta(): - arg[0] = mass_arg.get_logM() - else: - index_map.append(last_index) - bounds_list.append(mass_arg.get_logM_bounds()) - last_index += 1 - - if redshift_arg.is_dirac_delta(): - arg[1] = redshift_arg.get_z() - else: - index_map.append(last_index) - bounds_list.append(redshift_arg.get_z_bounds()) - last_index += 1 - - if mass_arg.dim > 0: - index_map += list(range(last_index, last_index + mass_arg.dim)) - bounds_list += mass_arg.get_proxy_bounds() - - last_index += mass_arg.dim - - if redshift_arg.dim > 0: - index_map += list(range(last_index, last_index + redshift_arg.dim)) - bounds_list += redshift_arg.get_proxy_bounds() - last_index += redshift_arg.dim - - if len(index_map) == 0: - # No proxy bins - return ( - self.mf_d2N_dz_dlnM(ccl_cosmo, arg[0], arg[1]) - * mass_arg.p(arg[0], arg[1]) - * redshift_arg.p(arg[0], arg[1]) - ) - - if self.prefer_scipy_integration: - return scipy.integrate.nquad( - integrand, - args=(index_map, arg, ccl_cosmo, mass_arg, redshift_arg), - ranges=bounds_list, - opts={"epsabs": self.abstol, "epsrel": self.reltol}, - )[0] - - Ncm.cfg_init() - int_nd = CountsIntegralND( - len(index_map), - integrand, - index_map, - arg, - ccl_cosmo, - mass_arg, - redshift_arg, - ) - int_nd.set_method(self.integ_method) - int_nd.set_reltol(self.reltol) - int_nd.set_abstol(self.abstol) - res = Ncm.Vector.new(1) - err = Ncm.Vector.new(1) - - bl, bu = zip(*bounds_list) - int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) - return res.get(0) - - def compute( - self, - ccl_cosmo: ccl.Cosmology, - mass_arg: ClusterMassArgument, - redshift_arg: ClusterRedshiftArgument, - ) -> float: - """Compute the integrand for the given cosmology at the given mass - and redshift.""" - return self._compute_any_from_args( - self._compute_integrand, ccl_cosmo, mass_arg, redshift_arg - ) - - def compute_unormalized_mean_logM( - self, - ccl_cosmo: ccl.Cosmology, - mass_arg: ClusterMassArgument, - redshift_arg: ClusterRedshiftArgument, - ): - """Compute the mean log(M) * integrand for the given cosmology at the - given mass and redshift. - """ - return self._compute_any_from_args( - self._compute_integrand_mean_logM, ccl_cosmo, mass_arg, redshift_arg - ) + def compute(self): + integrand = self.build_integrand() diff --git a/firecrown/models/cluster_mass.py b/firecrown/models/cluster_mass.py deleted file mode 100644 index a7efe8dc8..000000000 --- a/firecrown/models/cluster_mass.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Cluster Mass Module -abstract class to compute cluster mass function. -======================================== -The implemented functions use PyCCL library as backend. -""" -from __future__ import annotations -from typing import final, List, Tuple, Optional -from abc import abstractmethod - -import numpy as np -import sacc - -from ..updatable import Updatable -from ..parameters import ParamsMap - - -class ClusterMassArgument: - """Cluster Mass argument class.""" - - def __init__(self, logMl: float, logMu: float): - self.logMl: float = logMl - self.logMu: float = logMu - self.logM: Optional[float] = None - self.dirac_delta: bool = False - - if logMl > logMu: - raise ValueError("logMl must be smaller than logMu") - if logMl == logMu: - self.dirac_delta = True - self.logM = logMl - - def is_dirac_delta(self) -> bool: - """Check if the argument is a dirac delta.""" - - return self.dirac_delta - - def get_logM(self) -> float: - """Return the logM value if the argument is a dirac delta.""" - - if self.logM is not None: - return self.logM - raise ValueError("Argument is not a Dirac delta") - - @property - @abstractmethod - def dim(self) -> int: - """Return the dimension of the argument.""" - - @abstractmethod - def get_logM_bounds(self) -> Tuple[float, float]: - """Return the bounds of the cluster mass argument.""" - - @abstractmethod - def get_proxy_bounds(self) -> List[Tuple[float, float]]: - """Return the bounds of the cluster mass proxy argument.""" - - @abstractmethod - def p(self, logM: float, z: float, *proxy_args) -> float: - """Return the probability of the argument.""" - - -class ClusterMass(Updatable): - """Cluster Mass module.""" - - @abstractmethod - def read(self, sacc_data: sacc.Sacc): - """Abstract method to read the data for this source from the SACC file.""" - - def _update_cluster_mass(self, params: ParamsMap): - """Method to update the ClusterMass from the given ParamsMap. - Subclasses that need to do more than update their contained - :python:`Updatable` instance variables should implement this method.""" - - @final - def _update(self, params: ParamsMap): - """Implementation of Updatable interface method `_update`.""" - - self._update_cluster_mass(params) - - @abstractmethod - def gen_bins_by_array(self, logM_obs_bins: np.ndarray) -> List[ClusterMassArgument]: - """Generate bins by an array of bin edges.""" - - @abstractmethod - def gen_bin_from_tracer(self, tracer: sacc.BaseTracer) -> ClusterMassArgument: - """Return the bin for the given tracer.""" diff --git a/firecrown/models/cluster_mass_rich_proxy.py b/firecrown/models/cluster_mass_rich_proxy.py deleted file mode 100644 index 2b499b2cf..000000000 --- a/firecrown/models/cluster_mass_rich_proxy.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Cluster Mass Richness proxy module - -Define the Cluster Mass Richness proxy module and its arguments. -""" -from typing import List, Tuple, final - -import numpy as np -from scipy import special -import sacc - -from ..parameters import ( - ParamsMap, -) -from .cluster_mass import ClusterMass, ClusterMassArgument -from .. import parameters - - -class ClusterMassRich(ClusterMass): - """Cluster Mass Richness proxy. - - The following parameters are special Updatable parameters, which means that - they can be updated by the sampler: - - :ivar mu_p0: mu parameter 0 - :ivar mu_p1: mu parameter 1 - :ivar mu_p2: mu parameter 2 - :ivar sigma_p0: sigma parameter 0 - :ivar sigma_p1: sigma parameter 1 - :ivar sigma_p2: sigma parameter 2 - """ - - def __init__( - self, pivot_mass, pivot_redshift, logMl: float = 13.0, logMu: float = 16.0 - ): - """Initialize the ClusterMassRich object.""" - - super().__init__() - self.pivot_mass = pivot_mass - self.pivot_redshift = pivot_redshift - self.log_pivot_mass = pivot_mass * np.log(10.0) - self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) - self.logMl = logMl - self.logMu = logMu - - # Updatable parameters - self.mu_p0 = parameters.create() - self.mu_p1 = parameters.create() - self.mu_p2 = parameters.create() - self.sigma_p0 = parameters.create() - self.sigma_p1 = parameters.create() - self.sigma_p2 = parameters.create() - - self.logM_obs_min = 0.0 - self.logM_obs_max = np.inf - - @final - def _update_cluster_mass(self, params: ParamsMap): - """Perform any updates necessary after the parameters have being updated. - - This implementation has nothing to do.""" - - def read(self, _: sacc.Sacc): - """Method to read the data for this source from the SACC file.""" - - @staticmethod - def cluster_mass_parameters_function( - log_pivot_mass, log1p_pivot_redshift, p: Tuple[float, float, float], logM, z - ): - """Return observed quantity corrected by redshift and mass.""" - - lnM = logM * np.log(10) - Delta_lnM = lnM - log_pivot_mass - Delta_z = np.log1p(z) - log1p_pivot_redshift - - return p[0] + p[1] * Delta_lnM + p[2] * Delta_z - - def cluster_mass_lnM_obs_mu_sigma(self, logM, z): - """Return the mean and standard deviation of the observed mass.""" - - return [ - ClusterMassRich.cluster_mass_parameters_function( - self.log_pivot_mass, - self.log1p_pivot_redshift, - (self.mu_p0, self.mu_p1, self.mu_p2), - logM, - z, - ), - ClusterMassRich.cluster_mass_parameters_function( - self.log_pivot_mass, - self.log1p_pivot_redshift, - (self.sigma_p0, self.sigma_p1, self.sigma_p2), - logM, - z, - ), - ] - - def gen_bins_by_array(self, logM_obs_bins: np.ndarray) -> List[ClusterMassArgument]: - """Generate bins by an array of bin edges.""" - - if len(logM_obs_bins) < 2: - raise ValueError("logM_obs_bins must have at least two elements") - - # itertools.pairwise is only available in Python 3.10 - # using zip instead - return [ - ClusterMassRichBinArgument( - self, self.logMl, self.logMu, logM_obs_lower, logM_obs_upper - ) - for logM_obs_lower, logM_obs_upper in zip( - logM_obs_bins[:-1], logM_obs_bins[1:] - ) - ] - - def point_arg(self, logM_obs: float) -> ClusterMassArgument: - """Return the argument generator of the cluster mass function.""" - - return ClusterMassRichPointArgument(self, self.logMl, self.logMu, logM_obs) - - def gen_bin_from_tracer(self, tracer: sacc.BaseTracer) -> ClusterMassArgument: - """Return the argument for the given tracer.""" - - if not isinstance(tracer, sacc.tracers.BinRichnessTracer): - raise ValueError("Tracer must be a BinRichnessTracer") - - return ClusterMassRichBinArgument( - self, self.logMl, self.logMu, tracer.lower, tracer.upper - ) - - -class ClusterMassRichPointArgument(ClusterMassArgument): - """Argument for the Cluster Mass Richness proxy.""" - - def __init__( - self, - richness: ClusterMassRich, - logMl: float, - logMu: float, - logM_obs: float, - ): - super().__init__(logMl, logMu) - self.richness: ClusterMassRich = richness - self.logM_obs: float = logM_obs - - @property - def dim(self) -> int: - """Return the dimension of the argument.""" - return 0 - - def get_logM_bounds(self) -> Tuple[float, float]: - """Return the bounds of the cluster mass argument.""" - return (self.logMl, self.logMu) - - def get_proxy_bounds(self) -> List[Tuple[float, float]]: - """Return the bounds of the cluster mass proxy argument.""" - return [] - - def p(self, logM: float, z: float, *_) -> float: - """Return the probability of the point argument.""" - - lnM_obs = self.logM_obs * np.log(10.0) - - lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) - x = lnM_obs - lnM_mu - chisq = np.dot(x, x) / (2.0 * sigma**2) - likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) - return likelihood * np.log(10.0) - - -class ClusterMassRichBinArgument(ClusterMassArgument): - """Argument for the Cluster Mass Richness proxy.""" - - def __init__( - self, - richness: ClusterMassRich, - logMl: float, - logMu: float, - logM_obs_lower: float, - logM_obs_upper: float, - ): - super().__init__(logMl, logMu) - self.richness: ClusterMassRich = richness - self.logM_obs_lower: float = logM_obs_lower - self.logM_obs_upper: float = logM_obs_upper - if logM_obs_lower >= logM_obs_upper: - raise ValueError("logM_obs_lower must be less than logM_obs_upper") - - @property - def dim(self) -> int: - """Return the dimension of the argument.""" - return 0 - - def get_logM_bounds(self) -> Tuple[float, float]: - """Return the bounds of the cluster mass argument.""" - return (self.logMl, self.logMu) - - def get_proxy_bounds(self) -> List[Tuple[float, float]]: - """Return the bounds of the cluster mass proxy argument.""" - return [] - - def p(self, logM: float, z: float, *_) -> float: - """Return the probability of the binned argument.""" - - lnM_obs_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) - x_min = (lnM_obs_mu - self.logM_obs_lower * np.log(10.0)) / ( - np.sqrt(2.0) * sigma - ) - x_max = (lnM_obs_mu - self.logM_obs_upper * np.log(10.0)) / ( - np.sqrt(2.0) * sigma - ) - - if x_max > 3.0 or x_min < -3.0: - # pylint: disable-next=no-member - return -(special.erfc(x_min) - special.erfc(x_max)) / 2.0 - # pylint: disable-next=no-member - return (special.erf(x_min) - special.erf(x_max)) / 2.0 diff --git a/firecrown/models/cluster_mass_true.py b/firecrown/models/cluster_mass_true.py deleted file mode 100644 index 33d3adc38..000000000 --- a/firecrown/models/cluster_mass_true.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Cluster Mass True Module - -Class to compute cluster mass functions with no proxy, -i.e., assuming we have the true masses of the clusters. - -""" - -from typing import final, List, Tuple - -import numpy as np -import sacc - - -from ..parameters import ( - ParamsMap, -) -from .cluster_mass import ClusterMass, ClusterMassArgument - - -class ClusterMassTrue(ClusterMass): - """Cluster Mass class.""" - - @final - def _update_cluster_mass(self, params: ParamsMap): - """Method to update the ClusterMassTrue from the given ParamsMap.""" - - def read(self, sacc_data: sacc.Sacc): - """Method to read the data for this source from the SACC file.""" - - def gen_bins_by_array(self, logM_obs_bins: np.ndarray) -> List[ClusterMassArgument]: - """Generate the bins by an array of bin edges.""" - - if len(logM_obs_bins) < 2: - raise ValueError("logM_bins must have at least two elements") - - # itertools.pairwise is only available in Python 3.10 - # using zip instead - return [ - ClusterMassTrueArgument(lower, upper) - for lower, upper in zip(logM_obs_bins[:-1], logM_obs_bins[1:]) - ] - - def point_arg(self, logM: float) -> ClusterMassArgument: - """Return the argument for the given mass.""" - - return ClusterMassTrueArgument(logM, logM) - - def gen_bin_from_tracer(self, tracer: sacc.BaseTracer) -> ClusterMassArgument: - """Return the argument for the given tracer.""" - - if not isinstance(tracer, sacc.tracers.BinLogMTracer): - raise ValueError("Tracer must be a BinLogMTracer") - - return ClusterMassTrueArgument(tracer.lower, tracer.upper) - - -class ClusterMassTrueArgument(ClusterMassArgument): - """Cluster mass true argument class.""" - - @property - def dim(self) -> int: - """Return the dimension of the argument.""" - return 0 - - def get_logM_bounds(self) -> Tuple[float, float]: - """Return the bounds of the cluster mass argument.""" - return (self.logMl, self.logMu) - - def get_proxy_bounds(self) -> List[Tuple[float, float]]: - """Return the bounds of the cluster mass proxy argument.""" - return [] - - def p(self, logM: float, z: float, *_) -> float: - """Return the probability of the argument.""" - return 1.0 diff --git a/firecrown/models/cluster_redshift.py b/firecrown/models/cluster_redshift.py deleted file mode 100644 index ea9952fb1..000000000 --- a/firecrown/models/cluster_redshift.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Cluster Redshift Module -abstract class to compute cluster redshift functions. -======================================== -The implemented functions use PyCCL library as backend. -""" - -from typing import final, List, Tuple, Optional -from abc import abstractmethod - -import numpy as np - -import sacc - -from ..updatable import Updatable -from ..parameters import ParamsMap - - -class ClusterRedshiftArgument: - """Cluster Redshift argument class.""" - - def __init__(self, zl: float, zu: float): - self.zl: float = zl - self.zu: float = zu - self.z: Optional[float] = None - self.dirac_delta: bool = False - - if zl > zu: - raise ValueError("zl must be smaller than zu") - if zl == zu: - self.dirac_delta = True - self.z = zl - - def is_dirac_delta(self) -> bool: - """Check if the argument is a dirac delta.""" - - return self.dirac_delta - - def get_z(self) -> float: - """Return the z value if the argument is a dirac delta.""" - - if self.z is not None: - return self.z - raise ValueError("Argument is not a Dirac delta") - - @property - @abstractmethod - def dim(self) -> int: - """Return the dimension of the argument.""" - - @abstractmethod - def get_z_bounds(self) -> Tuple[float, float]: - """Return the bounds of the cluster redshift argument.""" - - @abstractmethod - def get_proxy_bounds(self) -> List[Tuple[float, float]]: - """Return the bounds of the cluster redshift proxy argument.""" - - @abstractmethod - def p(self, logM: float, z: float, *args) -> float: - """Return the probability of the argument.""" - - -class ClusterRedshift(Updatable): - """Cluster Redshift class.""" - - @abstractmethod - def read(self, sacc_data: sacc.Sacc): - """Abstract method to read the data for this source from the SACC file.""" - - def _update_cluster_redshift(self, params: ParamsMap): - """Method to update the ClusterRedshift from the given ParamsMap. - Subclasses that need to do more than update their contained - :python:`Updatable` instance variables should implement this method.""" - - @final - def _update(self, params: ParamsMap): - """Implementation of Updatable interface method `_update`.""" - - self._update_cluster_redshift(params) - - @abstractmethod - def gen_bins_by_array(self, z_bins: np.ndarray) -> List[ClusterRedshiftArgument]: - """Generate the bins by an array of bin edges.""" - - @abstractmethod - def gen_bin_from_tracer(self, tracer: sacc.BaseTracer) -> ClusterRedshiftArgument: - """Return the bin for the given tracer.""" diff --git a/firecrown/models/cluster_redshift_spec.py b/firecrown/models/cluster_redshift_spec.py deleted file mode 100644 index 32de42946..000000000 --- a/firecrown/models/cluster_redshift_spec.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Cluster Redshift Spectroscopy Module - -Class to compute cluster redshift spectroscopy functions. - -""" - -from typing import final, List, Tuple - -import numpy as np -import sacc - -from ..parameters import ( - ParamsMap, -) -from .cluster_redshift import ClusterRedshift, ClusterRedshiftArgument - - -class ClusterRedshiftSpec(ClusterRedshift): - """Cluster Redshift class.""" - - @final - def _update_cluster_redshift(self, params: ParamsMap): - """Method to update the ClusterRedshiftSpec from the given ParamsMap.""" - - def read(self, sacc_data: sacc.Sacc): - """Method to read the data for this source from the SACC file.""" - - def gen_bins_by_array(self, z_bins: np.ndarray) -> List[ClusterRedshiftArgument]: - """Generate the bins by an array of bin edges.""" - - if len(z_bins) < 2: - raise ValueError("z_bins must have at least two elements") - - # itertools.pairwise is only available in Python 3.10 - # using zip instead - return [ - ClusterRedshiftSpecArgument(lower, upper) - for lower, upper in zip(z_bins[:-1], z_bins[1:]) - ] - - def point_arg(self, z: float) -> ClusterRedshiftArgument: - """Return the argument for the given redshift.""" - - return ClusterRedshiftSpecArgument(z, z) - - def gen_bin_from_tracer(self, tracer: sacc.BaseTracer) -> ClusterRedshiftArgument: - """Return the argument for the given tracer.""" - - if not isinstance(tracer, sacc.tracers.BinZTracer): - raise ValueError("Tracer must be a BinZTracer") - - return ClusterRedshiftSpecArgument(tracer.lower, tracer.upper) - - -class ClusterRedshiftSpecArgument(ClusterRedshiftArgument): - """Cluster Redshift spectroscopy argument class.""" - - @property - def dim(self) -> int: - """Return the dimension of the argument.""" - return 0 - - def get_z_bounds(self) -> Tuple[float, float]: - """Return the bounds of the cluster redshift argument.""" - return (self.zl, self.zu) - - def get_proxy_bounds(self) -> List[Tuple[float, float]]: - """Return the bounds of the cluster redshift proxy argument.""" - return [] - - def p(self, logM: float, z: float, *args) -> float: - """Return the probability of the argument.""" - return 1.0 diff --git a/firecrown/models/cluster_theory/__init__.py b/firecrown/models/cluster_theory/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/firecrown/models/cluster_theory/cluster_abundance.py b/firecrown/models/cluster_theory/cluster_abundance.py deleted file mode 100644 index 950fb22ac..000000000 --- a/firecrown/models/cluster_theory/cluster_abundance.py +++ /dev/null @@ -1,73 +0,0 @@ -from pyccl.cosmology import Cosmology -import pyccl.background as bkg -import pyccl - - -class ClusterAbundance(object): - def __init__( - self, - cosmo: Cosmology, - halo_mass_function: pyccl.halos.MassFunc, - sky_area: float, - ): - self.cosmo = cosmo - self.halo_mass_function = halo_mass_function - self.sky_area = sky_area - - def update_ingredients(self, cosmo: Cosmology): - self.cosmo = cosmo - # Set cosmology dependent cluster abundance values. - pass - - def comoving_volume(self, z) -> float: - """Differential Comoving Volume at z. - - parameters - :param ccl_cosmo: pyccl Cosmology - :param z: Cluster Redshift. - - :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). - """ - scale_factor = 1.0 / (1.0 + z) - da = bkg.angular_diameter_distance(self.cosmo, scale_factor) - - h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) - dV = ( - ((1.0 + z) ** 2) - * (da**2) - * pyccl.physical_constants.CLIGHT_HMPC - / self.cosmo["h"] - / h_over_h0 - ) - return dV * self.sky_area_rad - - def mf_d2N_dz_dlnM_completeness(self, logM: float, z: float) -> float: - """ - Computes the mass function at z and logM. - - :param ccl_cosmo: pyccl Cosmology - :param logM: Cluster mass given by log10(M) where M is in units of - M_sun (comoving). - :param z: Cluster Redshift. - :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). - """ - mass_function = self.mass_funcion(self.cosmo, logM, z) - comoving_volume = self.comoving_volume(self.cosmo, z) - completeness = self._cluster_abundance_compute_completeness(logM, z) - - return mass_function * comoving_volume * completeness - - def mass_funcion(self, logM: float, z: float) -> float: - """ - Computes the mass function at z and logM. - - :param ccl_cosmo: pyccl Cosmology - :param logM: Cluster mass given by log10(M) where M is in units of - M_sun (comoving). - :param z: Cluster Redshift. - :return: Number Density pdf at z and logm in units of Mpc^-3 (comoving). - """ - scale_factor = 1.0 / (1.0 + z) - mass = 10 ** (logM) - nm = self.halo_mass_function(self.cosmo, mass, scale_factor) - return nm diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py new file mode 100644 index 000000000..dec3ee920 --- /dev/null +++ b/firecrown/models/kernel.py @@ -0,0 +1,49 @@ +from enum import Enum +import numpy as np +from firecrown.parameters import ParamsMap +from firecrown.updatable import Updatable + + +class Kernel(Updatable): + def __init__(): + super().__init__() + + def distribution(self, mass, z, mass_proxy, z_proxy): + return 1.0 + + +class Completeness(Kernel): + def __init__(): + super().__init__() + + # TODO get better names here + def distribution(self, mass, z, mass_proxy, z_proxy): + a_nc = 1.1321 + b_nc = 0.7751 + a_mc = 13.31 + b_mc = 0.2025 + log_mc = a_mc + b_mc * (1.0 + z) + nc = a_nc + b_nc * (1.0 + z) + completeness = (mass / log_mc) ** nc / ((mass / log_mc) ** nc + 1.0) + return completeness + + +class Purity(Kernel): + def __init__(): + super().__init__() + + # TODO get better names here + def distribution(mass, z, mass_proxy, z_proxy): + ln_r = np.log(10**mass_proxy) + a_nc = np.log(10) * 0.8612 + b_nc = np.log(10) * 0.3527 + a_rc = 2.2183 + b_rc = -0.6592 + nc = a_nc + b_nc * (1.0 + z) + ln_rc = a_rc + b_rc * (1.0 + z) + purity = (ln_r / ln_rc) ** nc / ((ln_r / ln_rc) ** nc + 1.0) + return purity + + +class Miscentering(Kernel): + pass diff --git a/firecrown/models/kernel_factory.py b/firecrown/models/kernel_factory.py new file mode 100644 index 000000000..bdc26bf50 --- /dev/null +++ b/firecrown/models/kernel_factory.py @@ -0,0 +1,21 @@ +from enum import Enum +import numpy as np +from firecrown.parameters import ParamsMap +from firecrown.updatable import Updatable +from .kernel import * + + +KernelType = Enum("KernelType", "COMPLETENESS PURITY MISCENTERING") + + +class KernelFactory: + @staticmethod + def create(KernelType: KernelType, params: ParamsMap = None): + if KernelType == KernelType.COMPLETENESS: + return Completeness(params) + elif KernelType == KernelType.PURITY: + return Purity(params) + elif KernelType == KernelType.MISCENTERING: + return Miscentering(params) + else: + raise ValueError(f"Kernel type {KernelType} not supported.") diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py new file mode 100644 index 000000000..174f0098a --- /dev/null +++ b/firecrown/models/mass_observable.py @@ -0,0 +1,100 @@ +"""Cluster Mass Richness proxy module + +Define the Cluster Mass Richness proxy module and its arguments. +""" +from typing import Tuple + +import numpy as np +from scipy import special +from ..parameters import ParamsMap +from .. import parameters +from .kernel import Kernel + + +class MassObservable(Kernel): + def __init__(self, params: ParamsMap): + """Initialize the ClusterMassRich object.""" + self.pivot_mass = params["pivot_mass"] + self.pivot_redshift = params["pivot_redshift"] + self.pivot_mass = self.pivot_mass * np.log(10.0) + + self.min_mass = params["min_mass"] + self.max_mass = params["max_mass"] + + self.min_obs_mass = 0.0 + self.max_obs_mass = np.inf + super().__init__() + + +class TrueMass(MassObservable): + def __init__(self, params: ParamsMap): + super().__init__(params) + + +class MassRichnessMuSigma(MassObservable): + def __init__(self, params: ParamsMap): + super().__init__(params) + + self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) + + # Updatable parameters + self.mu_p0 = parameters.create() + self.mu_p1 = parameters.create() + self.mu_p2 = parameters.create() + self.sigma_p0 = parameters.create() + self.sigma_p1 = parameters.create() + self.sigma_p2 = parameters.create() + + @staticmethod + def cluster_mass_parameters_function( + log_pivot_mass, log1p_pivot_redshift, p: Tuple[float, float, float], logM, z + ): + """Return observed quantity corrected by redshift and mass.""" + + lnM = logM * np.log(10) + Delta_lnM = lnM - log_pivot_mass + Delta_z = np.log1p(z) - log1p_pivot_redshift + + return p[0] + p[1] * Delta_lnM + p[2] * Delta_z + + def cluster_mass(self, mass, redshift): + return [ + MassRichnessMuSigma.cluster_mass_parameters_function( + self.pivot_mass, + self.log1p_pivot_redshift, + (self.mu_p0, self.mu_p1, self.mu_p2), + mass, + redshift, + ), + MassRichnessMuSigma.cluster_mass_parameters_function( + self.pivot_mass, + self.log1p_pivot_redshift, + (self.sigma_p0, self.sigma_p1, self.sigma_p2), + mass, + redshift, + ), + ] + + def distribution(self, mass, z, mass_proxy, z_proxy): + lnM_obs_mu, sigma = self.cluster_mass(mass, z) + + x_min = (lnM_obs_mu - self.min_obs_mass * np.log(10.0)) / (np.sqrt(2.0) * sigma) + x_max = (lnM_obs_mu - self.max_obs_mass * np.log(10.0)) / (np.sqrt(2.0) * sigma) + + if x_max > 3.0 or x_min < -3.0: + # pylint: disable-next=no-member + return -(special.erfc(x_min) - special.erfc(x_max)) / 2.0 + # pylint: disable-next=no-member + return (special.erf(x_min) - special.erf(x_max)) / 2.0 + + # TODO UNDERSTAND THIS + def spread_point(self, logM: float, z: float, *_) -> float: + """Return the probability of the point argument.""" + + lnM_obs = self.logM_obs * np.log(10.0) + + lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) + x = lnM_obs - lnM_mu + chisq = np.dot(x, x) / (2.0 * sigma**2) + likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) + return likelihood * np.log(10.0) diff --git a/firecrown/models/mass_observable_factory.py b/firecrown/models/mass_observable_factory.py new file mode 100644 index 000000000..decb47697 --- /dev/null +++ b/firecrown/models/mass_observable_factory.py @@ -0,0 +1,29 @@ +"""Cluster Mass Richness proxy module + +Define the Cluster Mass Richness proxy module and its arguments. +""" + +from ..parameters import ParamsMap +from enum import Enum +from .mass_observable import * + +MassObservableType = Enum("MassObservableType", "TRUE MU_SIGMA MURATA COSTANZI") + + +class MassObservableFactory: + @staticmethod + def create(mass_observable_type: MassObservableType, params: ParamsMap): + if mass_observable_type == MassObservableType.TRUE: + return TrueMass(params) + elif mass_observable_type == MassObservableType.MU_SIGMA: + return MassRichnessMuSigma(params) + elif mass_observable_type == MassObservableType.MURATA: + # return MurataMass(params) + raise NotImplementedError("Murata mass observable not implemented.") + elif mass_observable_type == MassObservableType.COSTANZI: + # return CostanziMass(params) + raise NotImplementedError("Costanzi mass observable not implemented.") + else: + raise ValueError( + f"Mass observable type {mass_observable_type} not supported." + ) diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py new file mode 100644 index 000000000..90516321b --- /dev/null +++ b/firecrown/models/redshift.py @@ -0,0 +1,32 @@ +import numpy as np +from firecrown.parameters import ParamsMap +from firecrown.models.kernel import Kernel + + +class Redshift(Kernel): + def __init__(self, params: ParamsMap = None): + super().__init__() + self.params = params + + def distribution(self, mass, z, mass_proxy, z_proxy): + pass + + +class SpectroscopicRedshiftUncertainty(Redshift): + def __init__(self, params: ParamsMap = None): + super().__init__(params) + + def distribution(self, mass, z, mass_proxy, z_proxy): + return 1.0 + + +class DESY1PhotometricRedshiftUncertainty(Redshift): + def __init__(self, params: ParamsMap = None): + super().__init__(params) + self.sigma_0 = 0.05 + + def distribution(self, mass, z, mass_proxy, z_proxy): + sigma_z = self.sigma_0 * (1 + z) + prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) + distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) + return prefactor * distribution diff --git a/firecrown/models/redshift_factory.py b/firecrown/models/redshift_factory.py new file mode 100644 index 000000000..08627b015 --- /dev/null +++ b/firecrown/models/redshift_factory.py @@ -0,0 +1,19 @@ +from enum import Enum +import numpy as np + +from firecrown.parameters import ParamsMap +from firecrown.models.kernel import Kernel +from .redshift import * + +RedshiftType = Enum("RedshiftType", "SPEC DESY1_PHOTO") + + +class RedshiftFactory: + @staticmethod + def create(redshift_type: RedshiftType, params: ParamsMap = None): + if redshift_type == RedshiftType.SPEC: + return SpectroscopicRedshiftUncertainty(params) + elif redshift_type == RedshiftType.DESY1_PHOTO: + return DESY1PhotometricRedshiftUncertainty(params) + else: + raise ValueError(f"Redshift type {redshift_type} not supported.") diff --git a/firecrown/models/sacc_adapter.py b/firecrown/models/sacc_adapter.py new file mode 100644 index 000000000..e577db30d --- /dev/null +++ b/firecrown/models/sacc_adapter.py @@ -0,0 +1,69 @@ +import sacc +import numpy as np +from sacc.tracers import SurveyTracer + + +class SaccAdapter: + # Hard coded in SACC, how do we want to handle this? + _survey_index = 0 + _mass_index = 1 + _redshift_index = 2 + + def __init__(self, sacc_data: sacc.Sacc): + self.sacc_data = sacc_data + + def get_data_and_indices(self, data_type, survey_tracer): + tracers_combinations = np.array( + self.sacc_data.get_tracer_combinations(data_type=data_type) + ) + self.validate_tracers(tracers_combinations, survey_tracer, data_type) + + cluster_survey_tracers = tracers_combinations[:, self._survey_index] + + self.z_tracers = tracers_combinations[:, self._redshift_index] + self.mass_tracers = tracers_combinations[:, self._mass_index] + self.survey_tracer = cluster_survey_tracers == survey_tracer + + data_vector_list = list( + self.sacc_data.get_mean(data_type=data_type)[self.survey_tracer] + ) + sacc_indices_list = list( + self.sacc_data.indices(data_type=data_type)[self.survey_tracer] + ) + return data_vector_list, sacc_indices_list + + def validate_tracers(self, tracers_combinations, survey_tracer, data_type): + if len(tracers_combinations) == 0: + raise ValueError( + f"The SACC file does not contain any tracers for the " + f"{data_type} data type." + ) + + if tracers_combinations.shape[1] != 3: + raise ValueError( + "The SACC file must contain 3 tracers for the " + "cluster_counts data type: cluster_survey, " + "redshift argument and mass argument tracers." + ) + cluster_survey_tracers = tracers_combinations[:, self._survey_index] + if survey_tracer not in cluster_survey_tracers: + raise ValueError( + f"The SACC tracer {self.survey_tracer} is not " + f"present in the SACC file." + ) + + def get_z_bins(self): + z_bins = {} + for z_tracer_nm in self.z_tracers: + tracer_data = self.sacc_data.get_tracer(z_tracer_nm) + z_bins[z_tracer_nm] = (tracer_data.lower, tracer_data.upper) + + return z_bins + + def get_mass_bins(self): + mass_bins = {} + for mass_tracer_nm in self.mass_tracers: + tracer_data = self.sacc_data.get_tracer(mass_tracer_nm) + mass_bins[mass_tracer_nm] = (tracer_data.lower, tracer_data.upper) + + return mass_bins diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 18a10bd99..c2c7d4586 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -11,7 +11,7 @@ from firecrown.models.cluster_mass import ClusterMass, ClusterMassArgument from firecrown.models.cluster_redshift import ClusterRedshift, ClusterRedshiftArgument from firecrown.models.cluster_mass_true import ClusterMassTrue -from firecrown.models.cluster_abundance import ClusterAbundance +from firecrown.models.cluster_abundance_old import ClusterAbundance from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec from firecrown.parameters import ParamsMap From b6336e5aa7956d159b26c76d7ef37995c439d06c Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 5 Oct 2023 14:35:41 -0700 Subject: [PATCH 03/80] Another round of commits. Stuck with the issue that the models in modelling tools also needs to know about the data (bounds for kernels) --- .../cluster_redshift_richness.py | 25 ++++-- .../statistic/cluster_number_counts.py | 59 +++++-------- firecrown/modeling_tools.py | 2 +- firecrown/models/cluster_abundance.py | 31 ++++--- firecrown/models/kernel.py | 17 ++-- firecrown/models/mass_observable.py | 85 +++++++++---------- firecrown/models/mass_observable_factory.py | 10 +-- firecrown/models/redshift.py | 6 +- firecrown/models/sacc_adapter.py | 57 +++++++------ 9 files changed, 156 insertions(+), 136 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 3f10a33d5..34fce33e3 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -12,19 +12,26 @@ from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster_abundance import ClusterAbundance -from firecrown.models.mass_observable import MassObservableFactory, MassObservableType -from firecrown.models.redshift import RedshiftFactory, RedshiftType +from firecrown.models.mass_observable_factory import ( + MassObservableFactory, + MassObservableType, +) +from firecrown.models.redshift_factory import RedshiftFactory, RedshiftType +from firecrown.models.kernel_factory import KernelFactory, KernelType from firecrown.parameters import ParamsMap +from firecrown.models.sacc_adapter import SaccAdapter def build_likelihood(build_parameters): """ Here we instantiate the number density (or mass function) object. """ + survey_name = "numcosmo_simulated_redshift_richness" + use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) + use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) + + stats = ClusterNumberCounts(survey_name, use_cluster_counts, use_mean_log_mass) - stats = ClusterNumberCounts(survey_name="numcosmo_simulated_redshift_richness") - stats.use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) - stats.use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) stats_list = [stats] # Here we instantiate the actual likelihood. The statistics argument carry @@ -40,6 +47,7 @@ def build_likelihood(build_parameters): ) ) sacc_data = sacc.Sacc.load_fits(saccfile) + sacc_adapter = SaccAdapter(sacc_data) # The read likelihood method is called passing the loaded SACC file, the # cluster number count functions will receive the appropriated sections @@ -51,6 +59,7 @@ def build_likelihood(build_parameters): # be used to compute the likelihood. hmf = ccl.halos.MassFuncTinker08() cluster_abundance = ClusterAbundance(hmf) + sa = SaccAdapter(sacc_data) mass_observable = MassObservableFactory.create( MassObservableType.MU_SIGMA, @@ -68,6 +77,12 @@ def build_likelihood(build_parameters): redshift = RedshiftFactory.create(RedshiftType.SPEC) cluster_abundance.add_kernel(redshift) + completeness = KernelFactory.create(KernelType.COMPLETENESS) + cluster_abundance.add_kernel(completeness) + + purity = KernelFactory.create(KernelType.PURITY) + cluster_abundance.add_kernel(purity) + modeling = ModelingTools(cluster_abundance=cluster_abundance) return lk, modeling diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index d23ff8cba..c8bf1f8b0 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -14,63 +14,48 @@ class ClusterNumberCounts(Statistic): - @property - def use_cluster_counts(self) -> bool: - return self._use_cluster_counts - - @property - def use_mean_log_mass(self) -> bool: - return self._use_mean_log_mass - - @use_cluster_counts.setter - def use_cluster_counts(self, value: bool): - self._use_cluster_counts = value - - @use_mean_log_mass.setter - def use_mean_log_mass(self, value: bool): - self._use_mean_log_mass = value - def __init__( self, + survey_name: str, + cluster_counts: bool, + mean_log_mass: bool, systematics: Optional[List[SourceSystematic]] = None, - survey_nm: str = "numcosmo_simulated_redshift_richness", ): super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None - self.survey_nm = survey_nm + self.survey_nm = survey_name + self.use_cluster_counts = cluster_counts + self.use_mean_log_mass = mean_log_mass def read(self, sacc_data: sacc.Sacc): - try: - survey_tracer: SurveyTracer = sacc_data.get_tracer(self.survey_nm) - except KeyError as exc: - raise ValueError( - f"The SACC file does not contain the SurveyTracer " f"{self.survey_nm}." - ) from exc - if not isinstance(survey_tracer, SurveyTracer): - raise ValueError(f"The SACC tracer {self.survey_nm} is not a SurveyTracer.") - sacc_types = sacc.data_types.standard_types - sa = SaccAdapter(sacc_data) + sa = SaccAdapter( + sacc_data, self.survey_nm, self.use_cluster_counts, self.use_mean_log_mass + ) data_vector_list = [] sacc_indices_list = [] + tracer_bounds = [] if self.use_cluster_counts: dtype = sacc_types.cluster_counts - data, indices = sa.get_data_and_indices(dtype, survey_tracer) + data, indices = sa.get_data_and_indices(dtype) data_vector_list += data sacc_indices_list += indices + tracer_bounds += sa.get_tracer_bounds(dtype) if self.use_mean_log_mass: dtype = sacc_types.cluster_mean_log_mass data, indices = sa.get_data_and_indices(dtype, survey_tracer) data_vector_list += data sacc_indices_list += indices + tracer_bounds += sa.get_tracer_bounds(dtype, survey_tracer) self.sky_area = sa.survey_tracer.sky_area self.data_vector = DataVector.from_list(data_vector_list) self.sacc_indices = np.array(sacc_indices_list) + self.tracer_bounds = np.array(tracer_bounds) super().read(sacc_data) def get_data_vector(self) -> DataVector: @@ -80,17 +65,17 @@ def get_data_vector(self) -> DataVector: def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: tools.cluster_abundance.sky_area = self.sky_area ccl_cosmo = tools.get_ccl_cosmology() - # tools.cluster_abundance.etc() + theory_vector_list = [] cluster_counts_list = [] - if self.use_cluster_counts or self.use_mean_log_mass: - cluster_counts_list = [ - self.cluster_abundance.compute(ccl_cosmo, logM_tracer_arg, z_tracer_arg) - for z_tracer_arg, logM_tracer_arg in self.tracer_args - ] - if self.use_cluster_counts: - theory_vector_list += cluster_counts_list + if self.use_cluster_counts: + for (z_min, z_max), (mproxy_min, mproxy_max) in self.tracer_bounds: + integrand = tools.cluster_abundance.get_abundance_integrand( + mass_min, z_min, mass_max, z_max + ) + + theory_vector_list += cluster_counts_list if self.use_mean_log_mass: mean_log_mass_list = [ diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index b9bb951ca..4ddf0bab2 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -7,7 +7,7 @@ from typing import Dict, Optional, final import pyccl.nl_pt -from .models.cluster_theory import ClusterAbundance +from .models.cluster_abundance import ClusterAbundance class ModelingTools: diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index c0250174a..50712aa1c 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -16,10 +16,7 @@ def sky_area(self) -> float: def sky_area(self, sky_area: float) -> None: self.sky_area_rad = sky_area * (np.pi / 180.0) ** 2 - def __init__( - self, - halo_mass_function: pyccl.halos.MassFunc, - ): + def __init__(self, halo_mass_function: pyccl.halos.MassFunc): self.kernels: List[Kernel] = [] self.cosmo = None self.halo_mass_function = halo_mass_function @@ -57,10 +54,24 @@ def mass_function(self, mass: float, z: float) -> float: hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) return hmf - def build_integrand(self, mass, z): - integrand = self.comoving_volume(z) * self.mass_function(mass, z) - for kernel in self.kernels: - integrand *= kernel.distribution(mass, z) + def get_abundance_integrand(self, mass, z): + def integrand(): + integrand = self.comoving_volume(z) * self.mass_function(mass, z) + for kernel in self.kernels: + integrand *= kernel.probability(mass, z) + return integrand - def compute(self): - integrand = self.build_integrand() + return integrand + + # Firecrown specific + def _process_args(self, args): + x = np.array(args[0:-5]) + index_map, arg, ccl_cosmo, mass_arg, redshift_arg = args[-5:] + arg[index_map] = x + redshift_start_index = 2 + redshift_arg.dim + + logM, z = arg[0:2] + proxy_z = arg[2:redshift_start_index] + proxy_m = arg[redshift_start_index:] + + return logM, z, proxy_z, proxy_m, ccl_cosmo, mass_arg, redshift_arg diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py index dec3ee920..f4c3fee36 100644 --- a/firecrown/models/kernel.py +++ b/firecrown/models/kernel.py @@ -5,19 +5,22 @@ class Kernel(Updatable): - def __init__(): + def __init__(self, bounds=[]): super().__init__() + # Number of differentials dx, dy, etc in the kernel + self.bounds = bounds # 2x dimension + self.dimension = len(bounds) - def distribution(self, mass, z, mass_proxy, z_proxy): + def probability(self, differentials): return 1.0 class Completeness(Kernel): - def __init__(): + def __init__(self): super().__init__() # TODO get better names here - def distribution(self, mass, z, mass_proxy, z_proxy): + def probability(self, mass, z, differentials): a_nc = 1.1321 b_nc = 0.7751 a_mc = 13.31 @@ -29,12 +32,12 @@ def distribution(self, mass, z, mass_proxy, z_proxy): class Purity(Kernel): - def __init__(): + def __init__(self): super().__init__() # TODO get better names here - def distribution(mass, z, mass_proxy, z_proxy): - ln_r = np.log(10**mass_proxy) + def probability(self, mass, z): + ln_r = np.log(10**mass) a_nc = np.log(10) * 0.8612 b_nc = np.log(10) * 0.3527 a_rc = 2.2183 diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py index 174f0098a..5d34cd9e8 100644 --- a/firecrown/models/mass_observable.py +++ b/firecrown/models/mass_observable.py @@ -18,11 +18,9 @@ def __init__(self, params: ParamsMap): self.pivot_redshift = params["pivot_redshift"] self.pivot_mass = self.pivot_mass * np.log(10.0) - self.min_mass = params["min_mass"] - self.max_mass = params["max_mass"] + self.min_mass = params["min_mass"] # 13 + self.max_mass = params["max_mass"] # 16 - self.min_obs_mass = 0.0 - self.max_obs_mass = np.inf super().__init__() @@ -32,7 +30,7 @@ def __init__(self, params: ParamsMap): class MassRichnessMuSigma(MassObservable): - def __init__(self, params: ParamsMap): + def __init__(self, params: ParamsMap, bounds): super().__init__(params) self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) @@ -46,40 +44,39 @@ def __init__(self, params: ParamsMap): self.sigma_p2 = parameters.create() @staticmethod - def cluster_mass_parameters_function( - log_pivot_mass, log1p_pivot_redshift, p: Tuple[float, float, float], logM, z + def observed_mass( + pivot_mass, log1p_pivot_redshift, p: Tuple[float, float, float], mass, z ): """Return observed quantity corrected by redshift and mass.""" - lnM = logM * np.log(10) - Delta_lnM = lnM - log_pivot_mass - Delta_z = np.log1p(z) - log1p_pivot_redshift - - return p[0] + p[1] * Delta_lnM + p[2] * Delta_z - - def cluster_mass(self, mass, redshift): - return [ - MassRichnessMuSigma.cluster_mass_parameters_function( - self.pivot_mass, - self.log1p_pivot_redshift, - (self.mu_p0, self.mu_p1, self.mu_p2), - mass, - redshift, - ), - MassRichnessMuSigma.cluster_mass_parameters_function( - self.pivot_mass, - self.log1p_pivot_redshift, - (self.sigma_p0, self.sigma_p1, self.sigma_p2), - mass, - redshift, - ), - ] - - def distribution(self, mass, z, mass_proxy, z_proxy): - lnM_obs_mu, sigma = self.cluster_mass(mass, z) - - x_min = (lnM_obs_mu - self.min_obs_mass * np.log(10.0)) / (np.sqrt(2.0) * sigma) - x_max = (lnM_obs_mu - self.max_obs_mass * np.log(10.0)) / (np.sqrt(2.0) * sigma) + ln_mass = mass * np.log(10) + delta_ln_mass = ln_mass - pivot_mass + delta_z = np.log1p(z) - log1p_pivot_redshift + + return p[0] + p[1] * delta_ln_mass + p[2] * delta_z + + def probability(self, mass, z): + observed_mean_mass = MassRichnessMuSigma.observed_mass( + self.pivot_mass, + self.log1p_pivot_redshift, + (self.mu_p0, self.mu_p1, self.mu_p2), + mass, + z, + ) + observed_mass_sigma = MassRichnessMuSigma.observed_mass( + self.pivot_mass, + self.log1p_pivot_redshift, + (self.sigma_p0, self.sigma_p1, self.sigma_p2), + mass, + z, + ) + + x_min = (observed_mean_mass - self.min_obs_mass * np.log(10.0)) / ( + np.sqrt(2.0) * observed_mass_sigma + ) + x_max = (observed_mean_mass - self.max_obs_mass * np.log(10.0)) / ( + np.sqrt(2.0) * observed_mass_sigma + ) if x_max > 3.0 or x_min < -3.0: # pylint: disable-next=no-member @@ -88,13 +85,13 @@ def distribution(self, mass, z, mass_proxy, z_proxy): return (special.erf(x_min) - special.erf(x_max)) / 2.0 # TODO UNDERSTAND THIS - def spread_point(self, logM: float, z: float, *_) -> float: - """Return the probability of the point argument.""" + # def spread_point(self, logM: float, z: float, *_) -> float: + # """Return the probability of the point argument.""" - lnM_obs = self.logM_obs * np.log(10.0) + # lnM_obs = self.logM_obs * np.log(10.0) - lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) - x = lnM_obs - lnM_mu - chisq = np.dot(x, x) / (2.0 * sigma**2) - likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) - return likelihood * np.log(10.0) + # lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) + # x = lnM_obs - lnM_mu + # chisq = np.dot(x, x) / (2.0 * sigma**2) + # likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) + # return likelihood * np.log(10.0) diff --git a/firecrown/models/mass_observable_factory.py b/firecrown/models/mass_observable_factory.py index decb47697..5794735b4 100644 --- a/firecrown/models/mass_observable_factory.py +++ b/firecrown/models/mass_observable_factory.py @@ -12,16 +12,16 @@ class MassObservableFactory: @staticmethod - def create(mass_observable_type: MassObservableType, params: ParamsMap): + def create(mass_observable_type: MassObservableType, params: ParamsMap, bounds): if mass_observable_type == MassObservableType.TRUE: - return TrueMass(params) + return TrueMass(params, bounds) elif mass_observable_type == MassObservableType.MU_SIGMA: - return MassRichnessMuSigma(params) + return MassRichnessMuSigma(params, bounds) elif mass_observable_type == MassObservableType.MURATA: - # return MurataMass(params) + # return MurataMass(params, bounds) raise NotImplementedError("Murata mass observable not implemented.") elif mass_observable_type == MassObservableType.COSTANZI: - # return CostanziMass(params) + # return CostanziMass(params, bounds) raise NotImplementedError("Costanzi mass observable not implemented.") else: raise ValueError( diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py index 90516321b..cc8b7fd85 100644 --- a/firecrown/models/redshift.py +++ b/firecrown/models/redshift.py @@ -8,7 +8,7 @@ def __init__(self, params: ParamsMap = None): super().__init__() self.params = params - def distribution(self, mass, z, mass_proxy, z_proxy): + def probability(self, mass, z, mass_proxy, z_proxy): pass @@ -16,7 +16,7 @@ class SpectroscopicRedshiftUncertainty(Redshift): def __init__(self, params: ParamsMap = None): super().__init__(params) - def distribution(self, mass, z, mass_proxy, z_proxy): + def probability(self, mass, z, mass_proxy, z_proxy): return 1.0 @@ -25,7 +25,7 @@ def __init__(self, params: ParamsMap = None): super().__init__(params) self.sigma_0 = 0.05 - def distribution(self, mass, z, mass_proxy, z_proxy): + def probability(self, mass, z, mass_proxy, z_proxy): sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) diff --git a/firecrown/models/sacc_adapter.py b/firecrown/models/sacc_adapter.py index e577db30d..01ab88ea8 100644 --- a/firecrown/models/sacc_adapter.py +++ b/firecrown/models/sacc_adapter.py @@ -9,21 +9,33 @@ class SaccAdapter: _mass_index = 1 _redshift_index = 2 - def __init__(self, sacc_data: sacc.Sacc): + def __init__( + self, sacc_data: sacc.Sacc, survey_nm: str, cluster_counts, mean_log_mass + ): self.sacc_data = sacc_data + self.cluster_counts = cluster_counts + self.mean_log_mass = mean_log_mass + try: + self.survey_tracer: SurveyTracer = sacc_data.get_tracer(survey_nm) + except KeyError as exc: + raise ValueError( + f"The SACC file does not contain the SurveyTracer " f"{survey_nm}." + ) from exc + if not isinstance(self.survey_tracer, SurveyTracer): + raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") - def get_data_and_indices(self, data_type, survey_tracer): + def filter_tracers(self, data_type): tracers_combinations = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) - self.validate_tracers(tracers_combinations, survey_tracer, data_type) - - cluster_survey_tracers = tracers_combinations[:, self._survey_index] + self.validate_tracers(tracers_combinations, data_type) - self.z_tracers = tracers_combinations[:, self._redshift_index] - self.mass_tracers = tracers_combinations[:, self._mass_index] - self.survey_tracer = cluster_survey_tracers == survey_tracer + self.z_tracers = np.unique(tracers_combinations[:, self._redshift_index]) + self.mass_tracers = np.unique(tracers_combinations[:, self._mass_index]) + self.survey_z_mass_tracers = tracers_combinations[self.survey_tracer] + def get_data_and_indices(self, data_type): + self.filter_tracers(data_type) data_vector_list = list( self.sacc_data.get_mean(data_type=data_type)[self.survey_tracer] ) @@ -32,7 +44,7 @@ def get_data_and_indices(self, data_type, survey_tracer): ) return data_vector_list, sacc_indices_list - def validate_tracers(self, tracers_combinations, survey_tracer, data_type): + def validate_tracers(self, tracers_combinations, data_type): if len(tracers_combinations) == 0: raise ValueError( f"The SACC file does not contain any tracers for the " @@ -45,25 +57,22 @@ def validate_tracers(self, tracers_combinations, survey_tracer, data_type): "cluster_counts data type: cluster_survey, " "redshift argument and mass argument tracers." ) - cluster_survey_tracers = tracers_combinations[:, self._survey_index] - if survey_tracer not in cluster_survey_tracers: - raise ValueError( - f"The SACC tracer {self.survey_tracer} is not " - f"present in the SACC file." - ) - def get_z_bins(self): - z_bins = {} + def get_tracer_bounds(self, data_type): + self.filter_tracers(data_type) + + z_bounds = {} for z_tracer_nm in self.z_tracers: tracer_data = self.sacc_data.get_tracer(z_tracer_nm) - z_bins[z_tracer_nm] = (tracer_data.lower, tracer_data.upper) - - return z_bins + z_bounds[z_tracer_nm] = (tracer_data.lower, tracer_data.upper) - def get_mass_bins(self): - mass_bins = {} + mass_bounds = {} for mass_tracer_nm in self.mass_tracers: tracer_data = self.sacc_data.get_tracer(mass_tracer_nm) - mass_bins[mass_tracer_nm] = (tracer_data.lower, tracer_data.upper) + mass_bounds[mass_tracer_nm] = (tracer_data.lower, tracer_data.upper) - return mass_bins + tracer_bounds = [ + (z_bounds[z_tracer], mass_bounds[mass_tracer]) + for _, z_tracer, mass_tracer in self.survey_z_mass_tracers + ] + return tracer_bounds From 6411a4fc091c0aa96797270958bc3b3ff0b60fa2 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 9 Oct 2023 09:11:06 -0700 Subject: [PATCH 04/80] flake8 is angry --- .../cluster_redshift_richness.py | 16 ++++++++-------- .../statistic/cluster_number_counts.py | 4 ++-- firecrown/models/kernel_factory.py | 2 +- firecrown/models/mass_observable_factory.py | 2 +- firecrown/models/redshift_factory.py | 5 ++++- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 34fce33e3..22e0233cc 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -61,7 +61,7 @@ def build_likelihood(build_parameters): cluster_abundance = ClusterAbundance(hmf) sa = SaccAdapter(sacc_data) - mass_observable = MassObservableFactory.create( + mass_observable_kernel = MassObservableFactory.create( MassObservableType.MU_SIGMA, ParamsMap( { @@ -72,16 +72,16 @@ def build_likelihood(build_parameters): } ), ) - cluster_abundance.add_kernel(mass_observable) + cluster_abundance.add_kernel(mass_observable_kernel) - redshift = RedshiftFactory.create(RedshiftType.SPEC) - cluster_abundance.add_kernel(redshift) + redshift_kernel = RedshiftFactory.create(RedshiftType.SPEC) + cluster_abundance.add_kernel(redshift_kernel) - completeness = KernelFactory.create(KernelType.COMPLETENESS) - cluster_abundance.add_kernel(completeness) + completeness_kernel = KernelFactory.create(KernelType.COMPLETENESS) + cluster_abundance.add_kernel(completeness_kernel) - purity = KernelFactory.create(KernelType.PURITY) - cluster_abundance.add_kernel(purity) + purity_kernel = KernelFactory.create(KernelType.PURITY) + cluster_abundance.add_kernel(purity_kernel) modeling = ModelingTools(cluster_abundance=cluster_abundance) diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index c8bf1f8b0..cff091a44 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -47,10 +47,10 @@ def read(self, sacc_data: sacc.Sacc): if self.use_mean_log_mass: dtype = sacc_types.cluster_mean_log_mass - data, indices = sa.get_data_and_indices(dtype, survey_tracer) + data, indices = sa.get_data_and_indices(dtype) data_vector_list += data sacc_indices_list += indices - tracer_bounds += sa.get_tracer_bounds(dtype, survey_tracer) + tracer_bounds += sa.get_tracer_bounds(dtype) self.sky_area = sa.survey_tracer.sky_area self.data_vector = DataVector.from_list(data_vector_list) diff --git a/firecrown/models/kernel_factory.py b/firecrown/models/kernel_factory.py index bdc26bf50..a1f8f729d 100644 --- a/firecrown/models/kernel_factory.py +++ b/firecrown/models/kernel_factory.py @@ -2,7 +2,7 @@ import numpy as np from firecrown.parameters import ParamsMap from firecrown.updatable import Updatable -from .kernel import * +from .kernel import Completeness, Purity, Miscentering KernelType = Enum("KernelType", "COMPLETENESS PURITY MISCENTERING") diff --git a/firecrown/models/mass_observable_factory.py b/firecrown/models/mass_observable_factory.py index 5794735b4..f93a52b80 100644 --- a/firecrown/models/mass_observable_factory.py +++ b/firecrown/models/mass_observable_factory.py @@ -5,7 +5,7 @@ from ..parameters import ParamsMap from enum import Enum -from .mass_observable import * +from .mass_observable import TrueMass, MassRichnessMuSigma MassObservableType = Enum("MassObservableType", "TRUE MU_SIGMA MURATA COSTANZI") diff --git a/firecrown/models/redshift_factory.py b/firecrown/models/redshift_factory.py index 08627b015..5e8720aee 100644 --- a/firecrown/models/redshift_factory.py +++ b/firecrown/models/redshift_factory.py @@ -3,7 +3,10 @@ from firecrown.parameters import ParamsMap from firecrown.models.kernel import Kernel -from .redshift import * +from .redshift import ( + SpectroscopicRedshiftUncertainty, + DESY1PhotometricRedshiftUncertainty, +) RedshiftType = Enum("RedshiftType", "SPEC DESY1_PHOTO") From 69d86c27da069d75975952414355f562b40c3eaa Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 9 Oct 2023 16:51:57 -0700 Subject: [PATCH 05/80] Removing factories in favor of explicit references. --- .../cluster_redshift_richness.py | 115 ++++++++++-------- firecrown/models/kernel_factory.py | 21 ---- firecrown/models/mass_observable_factory.py | 29 ----- firecrown/models/redshift_factory.py | 22 ---- 4 files changed, 65 insertions(+), 122 deletions(-) delete mode 100644 firecrown/models/kernel_factory.py delete mode 100644 firecrown/models/mass_observable_factory.py delete mode 100644 firecrown/models/redshift_factory.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 22e0233cc..d52b75fe2 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -1,88 +1,103 @@ """Likelihood factory function for cluster number counts.""" import os -from typing import Any, Dict +import numpy as np import pyccl as ccl import sacc from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( ClusterNumberCounts, + DataVector, ) from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster_abundance import ClusterAbundance -from firecrown.models.mass_observable_factory import ( - MassObservableFactory, - MassObservableType, -) -from firecrown.models.redshift_factory import RedshiftFactory, RedshiftType -from firecrown.models.kernel_factory import KernelFactory, KernelType -from firecrown.parameters import ParamsMap from firecrown.models.sacc_adapter import SaccAdapter +from firecrown.models.mass_observable import MassRichnessMuSigma +from firecrown.models.redshift import SpectroscopicRedshiftUncertainty +from firecrown.models.kernel import Completeness, Purity def build_likelihood(build_parameters): """ Here we instantiate the number density (or mass function) object. """ - survey_name = "numcosmo_simulated_redshift_richness" + + # Pull params for the likelihood from build_parameters use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) - stats = ClusterNumberCounts(survey_name, use_cluster_counts, use_mean_log_mass) - - stats_list = [stats] - - # Here we instantiate the actual likelihood. The statistics argument carry - # the order of the data/theory vector. - - lk = ConstGaussian(stats_list) + # Read in sacc data + sacc_file_nm = "cluster_redshift_richness_sacc_data.fits" + survey_name = "numcosmo_simulated_redshift_richness" + sacc_path = os.path.expanduser( + os.path.expandvars("${FIRECROWN_DIR}/examples/cluster_number_counts/") + ) + sacc_file = os.path.join(sacc_path, sacc_file_nm) - # We load the correct SACC file. - saccfile = os.path.expanduser( - os.path.expandvars( - "${FIRECROWN_DIR}/examples/cluster_number_counts/" - "cluster_redshift_richness_sacc_data.fits" - ) + sacc_data = sacc.Sacc.load_fits(sacc_file) + sacc_adapter = SaccAdapter( + sacc_data, survey_name, use_cluster_counts, use_mean_log_mass ) - sacc_data = sacc.Sacc.load_fits(saccfile) - sacc_adapter = SaccAdapter(sacc_data) - # The read likelihood method is called passing the loaded SACC file, the - # cluster number count functions will receive the appropriated sections - # of the SACC file. - lk.read(sacc_data) + # Build the data vector and indices needed for the likelihood + data_vector = [] + sacc_indices = [] + sacc_types = sacc.data_types.standard_types + if use_cluster_counts: + data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) + data_vector += data + sacc_indices += indices + + if use_mean_log_mass: + data, indices = sacc_adapter.get_data_and_indices( + sacc_types.cluster_mean_log_mass + ) + data_vector += data + sacc_indices += indices + + # Finally, build the statistics we want in our likelihood (cluster abundance) + counts = ClusterNumberCounts(use_cluster_counts, use_mean_log_mass) + counts.data_vector = DataVector.from_list(data_vector) + counts.sacc_indices = np.array(sacc_indices) + counts.sky_area = sacc_adapter.survey_tracer.sky_area + + mass_limits = [] + z_limits = [] + if use_cluster_counts: + mass_limits += sacc_adapter.get_mass_tracer_bin_limits( + sacc_types.cluster_counts + ) + z_limits += sacc_adapter.get_z_tracer_bin_limits(sacc_types.cluster_counts) + if use_mean_log_mass: + mass_limits += sacc_adapter.get_mass_tracer_bin_limits( + sacc_types.cluster_mean_log_mass + ) + z_limits += sacc_adapter.get_z_tracer_bin_limits( + sacc_types.cluster_mean_log_mass + ) - # This script will be loaded by the appropriated connector. The framework - # then looks for the `likelihood` variable to find the instance that will - # be used to compute the likelihood. hmf = ccl.halos.MassFuncTinker08() cluster_abundance = ClusterAbundance(hmf) - sa = SaccAdapter(sacc_data) - - mass_observable_kernel = MassObservableFactory.create( - MassObservableType.MU_SIGMA, - ParamsMap( - { - "pivot_mass": 14.625862906, - "pivot_redshift": 0.6, - "min_mass": 13.0, - "max_mass": 15.0, - } - ), + + # Create and add the kernels you want in your cluster abundance + mass_observable_kernel = MassRichnessMuSigma( + mass_limits, 14.625862906, 0.6, 13.0, 15.0 ) - cluster_abundance.add_kernel(mass_observable_kernel) + redshift_kernel = SpectroscopicRedshiftUncertainty() + completeness_kernel = Completeness() + purity_kernel = Purity() - redshift_kernel = RedshiftFactory.create(RedshiftType.SPEC) + cluster_abundance.add_kernel(mass_observable_kernel) cluster_abundance.add_kernel(redshift_kernel) - - completeness_kernel = KernelFactory.create(KernelType.COMPLETENESS) cluster_abundance.add_kernel(completeness_kernel) - - purity_kernel = KernelFactory.create(KernelType.PURITY) cluster_abundance.add_kernel(purity_kernel) + # Put it all together + stats_list = [counts] + lk = ConstGaussian(stats_list) + lk.read(sacc_data) modeling = ModelingTools(cluster_abundance=cluster_abundance) return lk, modeling diff --git a/firecrown/models/kernel_factory.py b/firecrown/models/kernel_factory.py deleted file mode 100644 index a1f8f729d..000000000 --- a/firecrown/models/kernel_factory.py +++ /dev/null @@ -1,21 +0,0 @@ -from enum import Enum -import numpy as np -from firecrown.parameters import ParamsMap -from firecrown.updatable import Updatable -from .kernel import Completeness, Purity, Miscentering - - -KernelType = Enum("KernelType", "COMPLETENESS PURITY MISCENTERING") - - -class KernelFactory: - @staticmethod - def create(KernelType: KernelType, params: ParamsMap = None): - if KernelType == KernelType.COMPLETENESS: - return Completeness(params) - elif KernelType == KernelType.PURITY: - return Purity(params) - elif KernelType == KernelType.MISCENTERING: - return Miscentering(params) - else: - raise ValueError(f"Kernel type {KernelType} not supported.") diff --git a/firecrown/models/mass_observable_factory.py b/firecrown/models/mass_observable_factory.py deleted file mode 100644 index f93a52b80..000000000 --- a/firecrown/models/mass_observable_factory.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Cluster Mass Richness proxy module - -Define the Cluster Mass Richness proxy module and its arguments. -""" - -from ..parameters import ParamsMap -from enum import Enum -from .mass_observable import TrueMass, MassRichnessMuSigma - -MassObservableType = Enum("MassObservableType", "TRUE MU_SIGMA MURATA COSTANZI") - - -class MassObservableFactory: - @staticmethod - def create(mass_observable_type: MassObservableType, params: ParamsMap, bounds): - if mass_observable_type == MassObservableType.TRUE: - return TrueMass(params, bounds) - elif mass_observable_type == MassObservableType.MU_SIGMA: - return MassRichnessMuSigma(params, bounds) - elif mass_observable_type == MassObservableType.MURATA: - # return MurataMass(params, bounds) - raise NotImplementedError("Murata mass observable not implemented.") - elif mass_observable_type == MassObservableType.COSTANZI: - # return CostanziMass(params, bounds) - raise NotImplementedError("Costanzi mass observable not implemented.") - else: - raise ValueError( - f"Mass observable type {mass_observable_type} not supported." - ) diff --git a/firecrown/models/redshift_factory.py b/firecrown/models/redshift_factory.py deleted file mode 100644 index 5e8720aee..000000000 --- a/firecrown/models/redshift_factory.py +++ /dev/null @@ -1,22 +0,0 @@ -from enum import Enum -import numpy as np - -from firecrown.parameters import ParamsMap -from firecrown.models.kernel import Kernel -from .redshift import ( - SpectroscopicRedshiftUncertainty, - DESY1PhotometricRedshiftUncertainty, -) - -RedshiftType = Enum("RedshiftType", "SPEC DESY1_PHOTO") - - -class RedshiftFactory: - @staticmethod - def create(redshift_type: RedshiftType, params: ParamsMap = None): - if redshift_type == RedshiftType.SPEC: - return SpectroscopicRedshiftUncertainty(params) - elif redshift_type == RedshiftType.DESY1_PHOTO: - return DESY1PhotometricRedshiftUncertainty(params) - else: - raise ValueError(f"Redshift type {redshift_type} not supported.") From b007840b876bb45fd79b8145862556726e2ac158 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 10 Oct 2023 09:36:57 -0700 Subject: [PATCH 06/80] Added some true mass and true redshift kernels Committing all work and going to start tests to check workflow. --- .../cluster_redshift_richness.py | 11 ++- .../statistic/cluster_number_counts.py | 68 ++++--------------- firecrown/models/cluster_abundance.py | 53 +++++++++++++-- firecrown/models/kernel.py | 55 ++++++++++----- firecrown/models/mass_observable.py | 55 ++++++++------- firecrown/models/redshift.py | 32 +++++---- firecrown/models/sacc_adapter.py | 27 ++++---- 7 files changed, 168 insertions(+), 133 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index d52b75fe2..e98bd2b80 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -14,8 +14,8 @@ from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster_abundance import ClusterAbundance from firecrown.models.sacc_adapter import SaccAdapter -from firecrown.models.mass_observable import MassRichnessMuSigma -from firecrown.models.redshift import SpectroscopicRedshiftUncertainty +from firecrown.models.mass_observable import MassRichnessMuSigma, Mass +from firecrown.models.redshift import SpectroscopicRedshift, Redshift from firecrown.models.kernel import Completeness, Purity @@ -82,15 +82,20 @@ def build_likelihood(build_parameters): cluster_abundance = ClusterAbundance(hmf) # Create and add the kernels you want in your cluster abundance + true_limits = [(0, np.inf)] + mass_kernel = Mass(true_limits) mass_observable_kernel = MassRichnessMuSigma( mass_limits, 14.625862906, 0.6, 13.0, 15.0 ) - redshift_kernel = SpectroscopicRedshiftUncertainty() + redshift_kernel = Redshift(true_limits) + redshift_proxy_kernel = SpectroscopicRedshift(z_limits) completeness_kernel = Completeness() purity_kernel = Purity() + cluster_abundance.add_kernel(mass_kernel) cluster_abundance.add_kernel(mass_observable_kernel) cluster_abundance.add_kernel(redshift_kernel) + cluster_abundance.add_kernel(redshift_proxy_kernel) cluster_abundance.add_kernel(completeness_kernel) cluster_abundance.add_kernel(purity_kernel) diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index cff091a44..0f80733ce 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -1,22 +1,14 @@ from __future__ import annotations -from typing import List, Dict, Tuple, Optional - -import numpy as np - +from typing import List, Optional import sacc -from sacc.tracers import SurveyTracer - from .statistic import Statistic, DataVector, TheoryVector from .source.source import SourceSystematic -from ....models.cluster_abundance import ClusterAbundance from ....modeling_tools import ModelingTools -from ....models.sacc_adapter import SaccAdapter class ClusterNumberCounts(Statistic): def __init__( self, - survey_name: str, cluster_counts: bool, mean_log_mass: bool, systematics: Optional[List[SourceSystematic]] = None, @@ -24,38 +16,12 @@ def __init__( super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None - self.survey_nm = survey_name self.use_cluster_counts = cluster_counts self.use_mean_log_mass = mean_log_mass + self.sky_area = 0.0 + self.data_vector = DataVector.from_list([]) def read(self, sacc_data: sacc.Sacc): - sacc_types = sacc.data_types.standard_types - sa = SaccAdapter( - sacc_data, self.survey_nm, self.use_cluster_counts, self.use_mean_log_mass - ) - - data_vector_list = [] - sacc_indices_list = [] - tracer_bounds = [] - - if self.use_cluster_counts: - dtype = sacc_types.cluster_counts - data, indices = sa.get_data_and_indices(dtype) - data_vector_list += data - sacc_indices_list += indices - tracer_bounds += sa.get_tracer_bounds(dtype) - - if self.use_mean_log_mass: - dtype = sacc_types.cluster_mean_log_mass - data, indices = sa.get_data_and_indices(dtype) - data_vector_list += data - sacc_indices_list += indices - tracer_bounds += sa.get_tracer_bounds(dtype) - - self.sky_area = sa.survey_tracer.sky_area - self.data_vector = DataVector.from_list(data_vector_list) - self.sacc_indices = np.array(sacc_indices_list) - self.tracer_bounds = np.array(tracer_bounds) super().read(sacc_data) def get_data_vector(self) -> DataVector: @@ -70,22 +36,18 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_counts_list = [] if self.use_cluster_counts: - for (z_min, z_max), (mproxy_min, mproxy_max) in self.tracer_bounds: - integrand = tools.cluster_abundance.get_abundance_integrand( - mass_min, z_min, mass_max, z_max - ) - + cluster_counts_list = tools.cluster_abundance.compute() theory_vector_list += cluster_counts_list - if self.use_mean_log_mass: - mean_log_mass_list = [ - self.cluster_abundance.compute_unormalized_mean_logM( - ccl_cosmo, logM_tracer_arg, z_tracer_arg - ) - / counts - for (z_tracer_arg, logM_tracer_arg), counts in zip( - self.tracer_args, cluster_counts_list - ) - ] - theory_vector_list += mean_log_mass_list + # if self.use_mean_log_mass: + # mean_log_mass_list = [ + # self.cluster_abundance.compute_unormalized_mean_logM( + # ccl_cosmo, logM_tracer_arg, z_tracer_arg + # ) + # / counts + # for (z_tracer_arg, logM_tracer_arg), counts in zip( + # self.tracer_args, cluster_counts_list + # ) + # ] + # theory_vector_list += mean_log_mass_list return TheoryVector.from_list(theory_vector_list) diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index 50712aa1c..7c652894a 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -1,7 +1,8 @@ -from typing import List +from typing import List, Dict from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl +from scipy.integrate import nquad from firecrown.models.kernel import Kernel import numpy as np @@ -37,12 +38,12 @@ def comoving_volume(self, z) -> float: :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). """ scale_factor = 1.0 / (1.0 + z) - da = bkg.angular_diameter_distance(self.cosmo, scale_factor) + angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) dV = ( ((1.0 + z) ** 2) - * (da**2) + * (angular_diam_dist**2) * pyccl.physical_constants.CLIGHT_HMPC / self.cosmo["h"] / h_over_h0 @@ -54,15 +55,55 @@ def mass_function(self, mass: float, z: float) -> float: hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) return hmf - def get_abundance_integrand(self, mass, z): - def integrand(): + def get_abundance_integrand(self): + def integrand(args: List[float], index_lookup: Dict[str, float]): + z = args[index_lookup["z"]] + mass = args[index_lookup["mass"]] + integrand = self.comoving_volume(z) * self.mass_function(mass, z) for kernel in self.kernels: - integrand *= kernel.probability(mass, z) + integrand *= kernel.distribution(args, index_lookup) return integrand return integrand + def get_abundance_bounds(self): + index_lookup = {} + bounds_list = [] + # Keep a separate index instead of using enumerate so we don't increment + # when we skip a dirac delta function + idx = 0 + for kernel in self.kernels: + if kernel.is_dirac_delta: + continue + index_lookup[kernel.kernel_type.name] = idx + bounds_list.append(kernel.integral_bounds) + idx += 1 + + return bounds_list, index_lookup + + def compute(self): + # Args = x0...xn, index_lkp = t0,...tm + # The function to be integrated. + # Has arguments of x0, ... xn, t0, ... tm, where integration is carried out + # over x0, ... xn, which must be floats. Where t0, ... tm are extra arguments + # passed in args. Function signature should be + # func(x0, x1, ..., xn, t0, t1, ..., tm). Integration is carried out in order. + # That is, integration over x0 is the innermost integral, and xn is the + # outermost. + integrand = self.get_abundance_integrand() + bounds_list, index_lookup = self.get_abundance_bounds() + return nquad( + # Function of the form f(x0, x1, ..., xn, t0, t1, ..., tm) + integrand, + # Each element of ranges must be a sequence of 2 numbers + # e.g. ranges[0] = (a,b) for f(x0, x1) + ranges=bounds_list, + # t0,...tn + args=(index_lookup), + opts={"epsabs": 1e-12, "epsrel": 1e-4}, + ) + # Firecrown specific def _process_args(self, args): x = np.array(args[0:-5]) diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py index f4c3fee36..2fb5fede3 100644 --- a/firecrown/models/kernel.py +++ b/firecrown/models/kernel.py @@ -1,26 +1,43 @@ -from enum import Enum +from typing import List, Tuple, Dict import numpy as np -from firecrown.parameters import ParamsMap from firecrown.updatable import Updatable +from abc import ABC, abstractmethod +from enum import Enum + +class KernelType(Enum): + mass = 1 + z = 2 + mass_proxy = 3 + z_proxy = 4 + completeness = 5 + purity = 6 -class Kernel(Updatable): - def __init__(self, bounds=[]): + +class Kernel(Updatable, ABC): + def __init__( + self, kernel_type: KernelType, integral_bounds: List[Tuple[float, float]] = None + ): super().__init__() - # Number of differentials dx, dy, etc in the kernel - self.bounds = bounds # 2x dimension - self.dimension = len(bounds) + self.integral_bounds = integral_bounds + self.is_dirac_delta = integral_bounds is None + self.kernel_type = kernel_type - def probability(self, differentials): - return 1.0 + # TODO change name to something that makes more sense for all proxies + # Spread? Distribution? + @abstractmethod + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + pass class Completeness(Kernel): - def __init__(self): - super().__init__() + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.completeness, integral_bounds) - # TODO get better names here - def probability(self, mass, z, differentials): + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + mass = args[index_lkp["mass"]] + z = args[index_lkp["z"]] + # TODO improve parameter names a_nc = 1.1321 b_nc = 0.7751 a_mc = 13.31 @@ -32,12 +49,14 @@ def probability(self, mass, z, differentials): class Purity(Kernel): - def __init__(self): - super().__init__() + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.purity, integral_bounds) - # TODO get better names here - def probability(self, mass, z): - ln_r = np.log(10**mass) + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + mass_proxy = args[index_lkp["mass_proxy"]] + z = args[index_lkp["z"]] + # TODO improve parameter names + ln_r = np.log(10**mass_proxy) a_nc = np.log(10) * 0.8612 b_nc = np.log(10) * 0.3527 a_rc = 2.2183 diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py index 5d34cd9e8..1c59c9ed2 100644 --- a/firecrown/models/mass_observable.py +++ b/firecrown/models/mass_observable.py @@ -2,37 +2,28 @@ Define the Cluster Mass Richness proxy module and its arguments. """ -from typing import Tuple +from typing import Tuple, List, Dict import numpy as np from scipy import special -from ..parameters import ParamsMap from .. import parameters -from .kernel import Kernel +from .kernel import Kernel, KernelType -class MassObservable(Kernel): - def __init__(self, params: ParamsMap): - """Initialize the ClusterMassRich object.""" - self.pivot_mass = params["pivot_mass"] - self.pivot_redshift = params["pivot_redshift"] +class MassRichnessMuSigma(Kernel): + def __init__( + self, + pivot_mass, + pivot_redshift, + min_mass=13.0, + max_mass=16.0, + integral_bounds: List[Tuple[float, float]] = None, + ): + self.pivot_mass = pivot_mass + self.pivot_redshift = pivot_redshift self.pivot_mass = self.pivot_mass * np.log(10.0) - - self.min_mass = params["min_mass"] # 13 - self.max_mass = params["max_mass"] # 16 - - super().__init__() - - -class TrueMass(MassObservable): - def __init__(self, params: ParamsMap): - super().__init__(params) - - -class MassRichnessMuSigma(MassObservable): - def __init__(self, params: ParamsMap, bounds): - super().__init__(params) - + self.min_mass = min_mass + self.max_mass = max_mass self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) # Updatable parameters @@ -43,6 +34,9 @@ def __init__(self, params: ParamsMap, bounds): self.sigma_p1 = parameters.create() self.sigma_p2 = parameters.create() + # Verify this gets called last or first + super().__init__(KernelType.mass_proxy, integral_bounds) + @staticmethod def observed_mass( pivot_mass, log1p_pivot_redshift, p: Tuple[float, float, float], mass, z @@ -55,7 +49,10 @@ def observed_mass( return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - def probability(self, mass, z): + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + mass = args[index_lkp["mass"]] + z = args[index_lkp["z"]] + observed_mean_mass = MassRichnessMuSigma.observed_mass( self.pivot_mass, self.log1p_pivot_redshift, @@ -95,3 +92,11 @@ def probability(self, mass, z): # chisq = np.dot(x, x) / (2.0 * sigma**2) # likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) # return likelihood * np.log(10.0) + + +class Mass(Kernel): + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.mass, integral_bounds) + + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + return 1.0 diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py index cc8b7fd85..8d5e47060 100644 --- a/firecrown/models/redshift.py +++ b/firecrown/models/redshift.py @@ -1,31 +1,33 @@ import numpy as np -from firecrown.parameters import ParamsMap -from firecrown.models.kernel import Kernel +from firecrown.models.kernel import Kernel, KernelType +from typing import List, Tuple, Dict class Redshift(Kernel): - def __init__(self, params: ParamsMap = None): - super().__init__() - self.params = params + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.z, integral_bounds) - def probability(self, mass, z, mass_proxy, z_proxy): - pass + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + return 1.0 -class SpectroscopicRedshiftUncertainty(Redshift): - def __init__(self, params: ParamsMap = None): - super().__init__(params) +class SpectroscopicRedshift(Kernel): + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.z_proxy, integral_bounds) - def probability(self, mass, z, mass_proxy, z_proxy): + def distribution(self, args: List[float], index_lkp: Dict[str, int]): return 1.0 -class DESY1PhotometricRedshiftUncertainty(Redshift): - def __init__(self, params: ParamsMap = None): - super().__init__(params) +class DESY1PhotometricRedshift(Kernel): + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.z_proxy, integral_bounds) self.sigma_0 = 0.05 - def probability(self, mass, z, mass_proxy, z_proxy): + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + z_proxy = args[index_lkp["z_proxy"]] + z = args[index_lkp["z"]] + sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) diff --git a/firecrown/models/sacc_adapter.py b/firecrown/models/sacc_adapter.py index 01ab88ea8..93d5ee42a 100644 --- a/firecrown/models/sacc_adapter.py +++ b/firecrown/models/sacc_adapter.py @@ -58,21 +58,22 @@ def validate_tracers(self, tracers_combinations, data_type): "redshift argument and mass argument tracers." ) - def get_tracer_bounds(self, data_type): + def get_mass_tracer_bin_limits(self, data_type): self.filter_tracers(data_type) - z_bounds = {} - for z_tracer_nm in self.z_tracers: - tracer_data = self.sacc_data.get_tracer(z_tracer_nm) - z_bounds[z_tracer_nm] = (tracer_data.lower, tracer_data.upper) + z_bounds = [ + (self.sacc_data.get_tracer(x).lower, self.sacc_data.get_tracer(x).upper) + for x in self.z_tracers + ] + + return z_bounds - mass_bounds = {} - for mass_tracer_nm in self.mass_tracers: - tracer_data = self.sacc_data.get_tracer(mass_tracer_nm) - mass_bounds[mass_tracer_nm] = (tracer_data.lower, tracer_data.upper) + def get_z_tracer_bin_limits(self, data_type): + self.filter_tracers(data_type) - tracer_bounds = [ - (z_bounds[z_tracer], mass_bounds[mass_tracer]) - for _, z_tracer, mass_tracer in self.survey_z_mass_tracers + mass_bounds = [ + (self.sacc_data.get_tracer(x).lower, self.sacc_data.get_tracer(x).upper) + for x in self.mass_tracers ] - return tracer_bounds + + return mass_bounds From ef6836474078a39252a0fb2a35bcfb3a266f55e6 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 10 Oct 2023 11:09:14 -0700 Subject: [PATCH 07/80] Added some beginning unit tests and fixed issues involving code. --- .../cluster_redshift_richness.py | 14 +- .../statistic/cluster_number_counts.py | 1 - firecrown/models/cluster_abundance.py | 15 +- firecrown/models/mass_observable.py | 2 +- firecrown/models/sacc_adapter.py | 33 +- .../statistic/test_cluster_number_counts.py | 122 +++---- tests/test_cluster.py | 250 ++++++------- tests/test_cluster_abundance.py | 79 +++++ tests/test_cluster_kernels.py | 98 ++++++ tests/test_clustermodel.py | 332 +++++++++--------- 10 files changed, 569 insertions(+), 377 deletions(-) create mode 100644 tests/test_cluster_abundance.py create mode 100644 tests/test_cluster_kernels.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index e98bd2b80..752349dcb 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -19,14 +19,17 @@ from firecrown.models.kernel import Completeness, Purity -def build_likelihood(build_parameters): +# def build_likelihood(build_parameters): +def build_likelihood(): """ Here we instantiate the number density (or mass function) object. """ # Pull params for the likelihood from build_parameters - use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) - use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) + # use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) + # use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) + use_cluster_counts = True + use_mean_log_mass = False # Read in sacc data sacc_file_nm = "cluster_redshift_richness_sacc_data.fits" @@ -85,7 +88,7 @@ def build_likelihood(build_parameters): true_limits = [(0, np.inf)] mass_kernel = Mass(true_limits) mass_observable_kernel = MassRichnessMuSigma( - mass_limits, 14.625862906, 0.6, 13.0, 15.0 + 14.625862906, 0.6, 13.0, 15.0, mass_limits ) redshift_kernel = Redshift(true_limits) redshift_proxy_kernel = SpectroscopicRedshift(z_limits) @@ -106,3 +109,6 @@ def build_likelihood(build_parameters): modeling = ModelingTools(cluster_abundance=cluster_abundance) return lk, modeling + + +build_likelihood() diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index 0f80733ce..d48076b62 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -30,7 +30,6 @@ def get_data_vector(self) -> DataVector: def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: tools.cluster_abundance.sky_area = self.sky_area - ccl_cosmo = tools.get_ccl_cosmology() theory_vector_list = [] cluster_counts_list = [] diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index 7c652894a..96456d5e3 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -17,16 +17,25 @@ def sky_area(self) -> float: def sky_area(self, sky_area: float) -> None: self.sky_area_rad = sky_area * (np.pi / 180.0) ** 2 - def __init__(self, halo_mass_function: pyccl.halos.MassFunc): + @property + def cosmo(self) -> Cosmology: + return self._cosmo + + def __init__( + self, + halo_mass_function: pyccl.halos.MassFunc, + sky_area_rad: float = 4 * np.pi**2, + ): self.kernels: List[Kernel] = [] - self.cosmo = None self.halo_mass_function = halo_mass_function + self.sky_area_rad = sky_area_rad + self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel): self.kernels.append(kernel) def update_ingredients(self, cosmo: Cosmology): - self.cosmo = cosmo + self._cosmo = cosmo def comoving_volume(self, z) -> float: """Differential Comoving Volume at z. diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py index 1c59c9ed2..f75123c8d 100644 --- a/firecrown/models/mass_observable.py +++ b/firecrown/models/mass_observable.py @@ -19,6 +19,7 @@ def __init__( max_mass=16.0, integral_bounds: List[Tuple[float, float]] = None, ): + super().__init__(KernelType.mass_proxy, integral_bounds) self.pivot_mass = pivot_mass self.pivot_redshift = pivot_redshift self.pivot_mass = self.pivot_mass * np.log(10.0) @@ -35,7 +36,6 @@ def __init__( self.sigma_p2 = parameters.create() # Verify this gets called last or first - super().__init__(KernelType.mass_proxy, integral_bounds) @staticmethod def observed_mass( diff --git a/firecrown/models/sacc_adapter.py b/firecrown/models/sacc_adapter.py index 93d5ee42a..2277311ca 100644 --- a/firecrown/models/sacc_adapter.py +++ b/firecrown/models/sacc_adapter.py @@ -6,8 +6,8 @@ class SaccAdapter: # Hard coded in SACC, how do we want to handle this? _survey_index = 0 - _mass_index = 1 - _redshift_index = 2 + _redshift_index = 1 + _mass_index = 2 def __init__( self, sacc_data: sacc.Sacc, survey_nm: str, cluster_counts, mean_log_mass @@ -17,6 +17,7 @@ def __init__( self.mean_log_mass = mean_log_mass try: self.survey_tracer: SurveyTracer = sacc_data.get_tracer(survey_nm) + self.survey_nm = survey_nm except KeyError as exc: raise ValueError( f"The SACC file does not contain the SurveyTracer " f"{survey_nm}." @@ -25,22 +26,22 @@ def __init__( raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") def filter_tracers(self, data_type): - tracers_combinations = np.array( + all_tracers = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) - self.validate_tracers(tracers_combinations, data_type) - - self.z_tracers = np.unique(tracers_combinations[:, self._redshift_index]) - self.mass_tracers = np.unique(tracers_combinations[:, self._mass_index]) - self.survey_z_mass_tracers = tracers_combinations[self.survey_tracer] + self.validate_tracers(all_tracers, data_type) + self.survey_mask = all_tracers[:, self._survey_index] == self.survey_nm + my_tracers = all_tracers[self.survey_mask] + self.z_tracers = np.unique(my_tracers[:, self._redshift_index]) + self.mass_tracers = np.unique(my_tracers[:, self._mass_index]) def get_data_and_indices(self, data_type): self.filter_tracers(data_type) data_vector_list = list( - self.sacc_data.get_mean(data_type=data_type)[self.survey_tracer] + self.sacc_data.get_mean(data_type=data_type)[self.survey_mask] ) sacc_indices_list = list( - self.sacc_data.indices(data_type=data_type)[self.survey_tracer] + self.sacc_data.indices(data_type=data_type)[self.survey_mask] ) return data_vector_list, sacc_indices_list @@ -61,19 +62,19 @@ def validate_tracers(self, tracers_combinations, data_type): def get_mass_tracer_bin_limits(self, data_type): self.filter_tracers(data_type) - z_bounds = [ + mass_bounds = [ (self.sacc_data.get_tracer(x).lower, self.sacc_data.get_tracer(x).upper) - for x in self.z_tracers + for x in self.mass_tracers ] - return z_bounds + return mass_bounds def get_z_tracer_bin_limits(self, data_type): self.filter_tracers(data_type) - mass_bounds = [ + z_bounds = [ (self.sacc_data.get_tracer(x).lower, self.sacc_data.get_tracer(x).upper) - for x in self.mass_tracers + for x in self.z_tracers ] - return mass_bounds + return z_bounds diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 4b11b3302..c4a91ddfb 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -1,61 +1,61 @@ -"""Tests for the ClusterNumberCounts statistic. -""" -import pytest - -import pyccl.halos -import sacc - -from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( - ClusterNumberCounts, - ClusterAbundance, -) -from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich -from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec - - -@pytest.fixture(name="minimal_stat") -def fixture_minimal_stat() -> ClusterNumberCounts: - """Return a correctly initialized :python:`ClusterNumberCounts` object.""" - stat = ClusterNumberCounts( - survey_tracer="SDSS", - cluster_abundance=ClusterAbundance( - halo_mass_definition=pyccl.halos.MassDef(0.5, "matter"), - halo_mass_function_name="200m", - halo_mass_function_args={}, - ), - cluster_mass=ClusterMassRich(pivot_mass=10.0, pivot_redshift=1.25), - cluster_redshift=ClusterRedshiftSpec(), - ) - return stat - - -@pytest.fixture(name="missing_survey_tracer") -def fixture_missing_survey_tracer() -> sacc.Sacc: - """Return a sacc.Sacc object that lacks a survey_tracer.""" - return sacc.Sacc() - - -@pytest.fixture(name="good_sacc_data") -def fixture_sacc_data(): - """Return a sacc.Sacc object sufficient to correctly set a - :python:`ClusterNumberCounts` object. - """ - data = sacc.Sacc() - return data - - -def test_missing_survey_tracer( - minimal_stat: ClusterNumberCounts, missing_survey_tracer: sacc.Sacc -): - with pytest.raises( - ValueError, match="The SACC file does not contain the SurveyTracer SDSS." - ): - minimal_stat.read(missing_survey_tracer) - - -def test_read_works(): - """After read() is called, we should be able to get the statistic's - - :python:`DataVector` and also should be able to call - :python:`compute_theory_vector`. - """ +# """Tests for the ClusterNumberCounts statistic. +# """ +# import pytest + +# import pyccl.halos +# import sacc + +# from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( +# ClusterNumberCounts, +# ClusterAbundance, +# ) +# from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich +# from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec + + +# @pytest.fixture(name="minimal_stat") +# def fixture_minimal_stat() -> ClusterNumberCounts: +# """Return a correctly initialized :python:`ClusterNumberCounts` object.""" +# stat = ClusterNumberCounts( +# survey_tracer="SDSS", +# cluster_abundance=ClusterAbundance( +# halo_mass_definition=pyccl.halos.MassDef(0.5, "matter"), +# halo_mass_function_name="200m", +# halo_mass_function_args={}, +# ), +# cluster_mass=ClusterMassRich(pivot_mass=10.0, pivot_redshift=1.25), +# cluster_redshift=ClusterRedshiftSpec(), +# ) +# return stat + + +# @pytest.fixture(name="missing_survey_tracer") +# def fixture_missing_survey_tracer() -> sacc.Sacc: +# """Return a sacc.Sacc object that lacks a survey_tracer.""" +# return sacc.Sacc() + + +# @pytest.fixture(name="good_sacc_data") +# def fixture_sacc_data(): +# """Return a sacc.Sacc object sufficient to correctly set a +# :python:`ClusterNumberCounts` object. +# """ +# data = sacc.Sacc() +# return data + + +# def test_missing_survey_tracer( +# minimal_stat: ClusterNumberCounts, missing_survey_tracer: sacc.Sacc +# ): +# with pytest.raises( +# ValueError, match="The SACC file does not contain the SurveyTracer SDSS." +# ): +# minimal_stat.read(missing_survey_tracer) + + +# def test_read_works(): +# """After read() is called, we should be able to get the statistic's + +# :python:`DataVector` and also should be able to call +# :python:`compute_theory_vector`. +# """ diff --git a/tests/test_cluster.py b/tests/test_cluster.py index c2c7d4586..abf5f6bcd 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,167 +1,167 @@ -"""Tests for the cluster module.""" +# """Tests for the cluster module.""" -from typing import Any, Dict, List -import itertools -import math +# from typing import Any, Dict, List +# import itertools +# import math -import pytest -import pyccl as ccl -import numpy as np +# import pytest +# import pyccl as ccl +# import numpy as np -from firecrown.models.cluster_mass import ClusterMass, ClusterMassArgument -from firecrown.models.cluster_redshift import ClusterRedshift, ClusterRedshiftArgument -from firecrown.models.cluster_mass_true import ClusterMassTrue -from firecrown.models.cluster_abundance_old import ClusterAbundance -from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich -from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec -from firecrown.parameters import ParamsMap +# from firecrown.models.cluster_mass import ClusterMass, ClusterMassArgument +# from firecrown.models.cluster_redshift import ClusterRedshift, ClusterRedshiftArgument +# from firecrown.models.cluster_mass_true import ClusterMassTrue +# from firecrown.models.cluster_abundance_old import ClusterAbundance +# from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich +# from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec +# from firecrown.parameters import ParamsMap -@pytest.fixture(name="ccl_cosmo") -def fixture_ccl_cosmo(): - """Fixture for a CCL cosmology object.""" +# @pytest.fixture(name="ccl_cosmo") +# def fixture_ccl_cosmo(): +# """Fixture for a CCL cosmology object.""" - return ccl.Cosmology( - Omega_c=0.22, Omega_b=0.0448, h=0.71, sigma8=0.8, n_s=0.963, Neff=3.44 - ) +# return ccl.Cosmology( +# Omega_c=0.22, Omega_b=0.0448, h=0.71, sigma8=0.8, n_s=0.963, Neff=3.44 +# ) -@pytest.fixture(name="parameters") -def fixture_parameters(): - """Fixture for a parameter map.""" +# @pytest.fixture(name="parameters") +# def fixture_parameters(): +# """Fixture for a parameter map.""" - parameters = ParamsMap( - { - "mu_p0": 3.19, - "mu_p1": 0.8, - "mu_p2": 0.0, - "sigma_p0": 0.3, - "sigma_p1": 0.08, - "sigma_p2": 0.0, - } - ) - return parameters +# parameters = ParamsMap( +# { +# "mu_p0": 3.19, +# "mu_p1": 0.8, +# "mu_p2": 0.0, +# "sigma_p0": 0.3, +# "sigma_p1": 0.08, +# "sigma_p2": 0.0, +# } +# ) +# return parameters -@pytest.fixture(name="z_args") -def fixture_cluster_z_args(parameters): - """Fixture for cluster redshifts.""" - z_bins = np.array([0.2000146, 0.31251036, 0.42500611, 0.53750187, 0.64999763]) - cluster_z = ClusterRedshiftSpec() - assert isinstance(cluster_z, ClusterRedshift) +# @pytest.fixture(name="z_args") +# def fixture_cluster_z_args(parameters): +# """Fixture for cluster redshifts.""" +# z_bins = np.array([0.2000146, 0.31251036, 0.42500611, 0.53750187, 0.64999763]) +# cluster_z = ClusterRedshiftSpec() +# assert isinstance(cluster_z, ClusterRedshift) - z_args = cluster_z.gen_bins_by_array(z_bins) +# z_args = cluster_z.gen_bins_by_array(z_bins) - cluster_z.update(parameters) +# cluster_z.update(parameters) - return z_args +# return z_args -@pytest.fixture(name="logM_args") -def fixture_cluster_mass_logM_args(parameters): - """Fixture for cluster masses.""" - logM_bins = np.array([13.0, 13.5, 14.0, 14.5, 15.0]) - cluster_mass_t = ClusterMassTrue() - assert isinstance(cluster_mass_t, ClusterMass) +# @pytest.fixture(name="logM_args") +# def fixture_cluster_mass_logM_args(parameters): +# """Fixture for cluster masses.""" +# logM_bins = np.array([13.0, 13.5, 14.0, 14.5, 15.0]) +# cluster_mass_t = ClusterMassTrue() +# assert isinstance(cluster_mass_t, ClusterMass) - logM_args = cluster_mass_t.gen_bins_by_array(logM_bins) - cluster_mass_t.update(parameters) +# logM_args = cluster_mass_t.gen_bins_by_array(logM_bins) +# cluster_mass_t.update(parameters) - return logM_args +# return logM_args -@pytest.fixture(name="rich_args") -def fixture_cluster_mass_rich_args(parameters): - """Fixture for cluster masses.""" - pivot_mass = 14.0 - pivot_redshift = 0.6 - proxy_bins = np.array([0.45805137, 0.81610273, 1.1741541, 1.53220547, 1.89025684]) +# @pytest.fixture(name="rich_args") +# def fixture_cluster_mass_rich_args(parameters): +# """Fixture for cluster masses.""" +# pivot_mass = 14.0 +# pivot_redshift = 0.6 +# proxy_bins = np.array([0.45805137, 0.81610273, 1.1741541, 1.53220547, 1.89025684]) - cluster_mass_r = ClusterMassRich(pivot_mass, pivot_redshift) - assert isinstance(cluster_mass_r, ClusterMass) +# cluster_mass_r = ClusterMassRich(pivot_mass, pivot_redshift) +# assert isinstance(cluster_mass_r, ClusterMass) - rich_args = cluster_mass_r.gen_bins_by_array(proxy_bins) - cluster_mass_r.update(parameters) +# rich_args = cluster_mass_r.gen_bins_by_array(proxy_bins) +# cluster_mass_r.update(parameters) - return rich_args +# return rich_args -@pytest.fixture(name="cluster_abundance") -def fixture_cluster_abundance(parameters): - """Fixture for cluster objects.""" +# @pytest.fixture(name="cluster_abundance") +# def fixture_cluster_abundance(parameters): +# """Fixture for cluster objects.""" - # TODO: remove try/except when pyccl 3.0 is released - try: - hmd_200 = ccl.halos.MassDef200c() - except TypeError: - hmd_200 = ccl.halos.MassDef200c +# # TODO: remove try/except when pyccl 3.0 is released +# try: +# hmd_200 = ccl.halos.MassDef200c() +# except TypeError: +# hmd_200 = ccl.halos.MassDef200c - hmf_args: Dict[str, Any] = {} - hmf_name = "Bocquet16" - sky_area = 489 +# hmf_args: Dict[str, Any] = {} +# hmf_name = "Bocquet16" +# sky_area = 489 - cluster_abundance = ClusterAbundance(hmd_200, hmf_name, hmf_args, sky_area) - assert isinstance(cluster_abundance, ClusterAbundance) +# cluster_abundance = ClusterAbundance(hmd_200, hmf_name, hmf_args, sky_area) +# assert isinstance(cluster_abundance, ClusterAbundance) - cluster_abundance.update(parameters) +# cluster_abundance.update(parameters) - return cluster_abundance +# return cluster_abundance -def test_initialize_objects( - ccl_cosmo: ccl.Cosmology, cluster_abundance, z_args, logM_args, rich_args -): - """Test initialization of cluster objects.""" +# def test_initialize_objects( +# ccl_cosmo: ccl.Cosmology, cluster_abundance, z_args, logM_args, rich_args +# ): +# """Test initialization of cluster objects.""" - for z_arg in z_args: - assert isinstance(z_arg, ClusterRedshiftArgument) +# for z_arg in z_args: +# assert isinstance(z_arg, ClusterRedshiftArgument) - for logM_arg in logM_args: - assert isinstance(logM_arg, ClusterMassArgument) +# for logM_arg in logM_args: +# assert isinstance(logM_arg, ClusterMassArgument) - for rich_arg in rich_args: - assert isinstance(rich_arg, ClusterMassArgument) +# for rich_arg in rich_args: +# assert isinstance(rich_arg, ClusterMassArgument) - assert isinstance(ccl_cosmo, ccl.Cosmology) - assert isinstance(cluster_abundance, ClusterAbundance) +# assert isinstance(ccl_cosmo, ccl.Cosmology) +# assert isinstance(cluster_abundance, ClusterAbundance) -def test_cluster_mass_function_compute( - ccl_cosmo: ccl.Cosmology, - cluster_abundance: ClusterAbundance, - z_args: List[ClusterRedshiftArgument], - logM_args: List[ClusterMassArgument], - rich_args: List[ClusterMassArgument], -): - """Test cluster mass function computations.""" +# def test_cluster_mass_function_compute( +# ccl_cosmo: ccl.Cosmology, +# cluster_abundance: ClusterAbundance, +# z_args: List[ClusterRedshiftArgument], +# logM_args: List[ClusterMassArgument], +# rich_args: List[ClusterMassArgument], +# ): +# """Test cluster mass function computations.""" - for redshift_arg, logM_arg, rich_arg in itertools.product( - z_args, logM_args, rich_args - ): - count_rich = cluster_abundance.compute(ccl_cosmo, rich_arg, redshift_arg) - assert math.isfinite(count_rich) - assert count_rich > 0.0 - count_mass = cluster_abundance.compute(ccl_cosmo, logM_arg, redshift_arg) - assert math.isfinite(count_mass) - assert count_mass > 0.0 +# for redshift_arg, logM_arg, rich_arg in itertools.product( +# z_args, logM_args, rich_args +# ): +# count_rich = cluster_abundance.compute(ccl_cosmo, rich_arg, redshift_arg) +# assert math.isfinite(count_rich) +# assert count_rich > 0.0 +# count_mass = cluster_abundance.compute(ccl_cosmo, logM_arg, redshift_arg) +# assert math.isfinite(count_mass) +# assert count_mass > 0.0 -def test_cluster_mass_function_compute_scipy( - ccl_cosmo: ccl.Cosmology, - cluster_abundance: ClusterAbundance, - z_args: List[ClusterRedshiftArgument], - logM_args: List[ClusterMassArgument], - rich_args: List[ClusterMassArgument], -): - """Test cluster mass function computations.""" +# def test_cluster_mass_function_compute_scipy( +# ccl_cosmo: ccl.Cosmology, +# cluster_abundance: ClusterAbundance, +# z_args: List[ClusterRedshiftArgument], +# logM_args: List[ClusterMassArgument], +# rich_args: List[ClusterMassArgument], +# ): +# """Test cluster mass function computations.""" - for redshift_arg, logM_arg, rich_arg in itertools.product( - z_args, logM_args, rich_args - ): - cluster_abundance.prefer_scipy_integration = True - count_rich = cluster_abundance.compute(ccl_cosmo, rich_arg, redshift_arg) - assert math.isfinite(count_rich) - assert count_rich > 0.0 - count_mass = cluster_abundance.compute(ccl_cosmo, logM_arg, redshift_arg) - assert math.isfinite(count_mass) - assert count_mass > 0.0 +# for redshift_arg, logM_arg, rich_arg in itertools.product( +# z_args, logM_args, rich_args +# ): +# cluster_abundance.prefer_scipy_integration = True +# count_rich = cluster_abundance.compute(ccl_cosmo, rich_arg, redshift_arg) +# assert math.isfinite(count_rich) +# assert count_rich > 0.0 +# count_mass = cluster_abundance.compute(ccl_cosmo, logM_arg, redshift_arg) +# assert math.isfinite(count_mass) +# assert count_mass > 0.0 diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py new file mode 100644 index 000000000..0c97d2478 --- /dev/null +++ b/tests/test_cluster_abundance.py @@ -0,0 +1,79 @@ +from firecrown.models.cluster_abundance import ClusterAbundance +from firecrown.models.kernel import Kernel, KernelType +import pyccl +import numpy as np +from typing import List, Tuple, Dict + + +class MockKernel(Kernel): + def __init__(self, integral_bounds: List[Tuple[float, float]] = None): + super().__init__(KernelType.mass, integral_bounds) + + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + return 1.0 + + +def test_cluster_abundance_init(): + hmf = pyccl.halos.MassFuncTinker08() + ca = ClusterAbundance(hmf) + assert ca is not None + assert ca.cosmo is None + assert isinstance(ca.halo_mass_function, pyccl.halos.MassFuncTinker08) + + +def test_cluster_update_ingredients(): + hmf = pyccl.halos.MassFuncTinker08() + ca = ClusterAbundance(hmf) + + assert ca.cosmo is None + cosmo = pyccl.CosmologyVanillaLCDM() + + ca.update_ingredients(cosmo) + assert ca.cosmo is not None + assert ca.cosmo == cosmo + + +def test_cluster_sky_area(): + hmf = pyccl.halos.MassFuncTinker08() + ca = ClusterAbundance(hmf) + assert ca.sky_area == 360.0**2 + assert ca.sky_area_rad == 4 * np.pi**2 + + ca.sky_area = 180.0**2 + assert ca.sky_area == 180.0**2 + assert ca.sky_area_rad == np.pi**2 + + +def test_cluster_add_kernel(): + hmf = pyccl.halos.MassFuncTinker08() + ca = ClusterAbundance(hmf) + assert len(ca.kernels) == 0 + + ca.add_kernel(MockKernel()) + assert len(ca.kernels) == 1 + assert isinstance(ca.kernels[0], Kernel) + + +def test_cluster_abundance_bounds(): + hmf = pyccl.halos.MassFuncTinker08() + ca = ClusterAbundance(hmf) + bounds, index_lookup = ca.get_abundance_bounds() + assert len(bounds) == 0 + assert len(index_lookup) == 0 + + ca.add_kernel(MockKernel()) + bounds, index_lookup = ca.get_abundance_bounds() + assert len(bounds) == 0 + assert len(index_lookup) == 0 + + ca.add_kernel(MockKernel([(0, 1)])) + bounds, index_lookup = ca.get_abundance_bounds() + assert len(bounds) == 1 + assert len(index_lookup) == 1 + assert index_lookup["mass"] == 0 + assert bounds[index_lookup["mass"]] == [(0, 1)] + + +def test_cluster_abundance_integrand(): + hmf = pyccl.halos.MassFuncTinker08() + ca = ClusterAbundance(hmf) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py new file mode 100644 index 000000000..d30ab3e7d --- /dev/null +++ b/tests/test_cluster_kernels.py @@ -0,0 +1,98 @@ +from firecrown.models.kernel import Completeness, Purity, KernelType, Kernel +from firecrown.models.mass_observable import Mass, MassRichnessMuSigma +from firecrown.models.redshift import ( + Redshift, + SpectroscopicRedshift, + DESY1PhotometricRedshift, +) + + +def test_desy1_photometric_redshift_kernel(): + drk = DESY1PhotometricRedshift() + assert isinstance(drk, Kernel) + assert drk.kernel_type == KernelType.z_proxy + assert drk.is_dirac_delta is True + assert drk.integral_bounds is None + + drk = DESY1PhotometricRedshift([(0, 1)]) + assert drk.is_dirac_delta is False + assert len(drk.integral_bounds) == 1 + assert drk.integral_bounds[0] == (0, 1) + + +def test_spectroscopic_redshift_kernel(): + srk = SpectroscopicRedshift() + assert isinstance(srk, Kernel) + assert srk.kernel_type == KernelType.z_proxy + assert srk.is_dirac_delta is True + assert srk.integral_bounds is None + + srk = SpectroscopicRedshift([(0, 1)]) + assert srk.is_dirac_delta is False + assert len(srk.integral_bounds) == 1 + assert srk.integral_bounds[0] == (0, 1) + + +def test_redshift_kernel(): + rk = Redshift() + assert isinstance(rk, Kernel) + assert rk.kernel_type == KernelType.z + assert rk.is_dirac_delta is True + assert rk.integral_bounds is None + + rk = Redshift([(0, 1)]) + assert rk.is_dirac_delta is False + assert len(rk.integral_bounds) == 1 + assert rk.integral_bounds[0] == (0, 1) + + +def test_musigma_kernel(): + msk = MassRichnessMuSigma(1, 1) + assert isinstance(msk, Kernel) + assert msk.kernel_type == KernelType.mass_proxy + assert msk.is_dirac_delta is True + assert msk.integral_bounds is None + + msk = MassRichnessMuSigma(1, 1, integral_bounds=[(0, 1)]) + assert msk.is_dirac_delta is False + assert len(msk.integral_bounds) == 1 + assert msk.integral_bounds[0] == (0, 1) + + +def test_mass_kernel(): + mk = Mass() + assert isinstance(mk, Kernel) + assert mk.kernel_type == KernelType.mass + assert mk.is_dirac_delta is True + assert mk.integral_bounds is None + + mk = Mass([(0, 1)]) + assert mk.is_dirac_delta is False + assert len(mk.integral_bounds) == 1 + assert mk.integral_bounds[0] == (0, 1) + + +def test_completeness_kernel(): + ck = Completeness() + assert isinstance(ck, Kernel) + assert ck.kernel_type == KernelType.completeness + assert ck.is_dirac_delta is True + assert ck.integral_bounds is None + + ck = Completeness([(0, 1)]) + assert ck.is_dirac_delta is False + assert len(ck.integral_bounds) == 1 + assert ck.integral_bounds[0] == (0, 1) + + +def test_purity_kernel(): + pk = Purity() + assert isinstance(pk, Kernel) + assert pk.kernel_type == KernelType.purity + assert pk.is_dirac_delta is True + assert pk.integral_bounds is None + + pk = Purity([(0, 1)]) + assert pk.is_dirac_delta is False + assert len(pk.integral_bounds) == 1 + assert pk.integral_bounds[0] == (0, 1) diff --git a/tests/test_clustermodel.py b/tests/test_clustermodel.py index 2da4cc0fe..99cc492ad 100644 --- a/tests/test_clustermodel.py +++ b/tests/test_clustermodel.py @@ -1,168 +1,168 @@ -"""Test cluster mass function computations.""" - -import itertools -import numpy as np -import pytest +# """Test cluster mass function computations.""" + +# import itertools +# import numpy as np +# import pytest -from firecrown.models.cluster_mass_rich_proxy import ( - ClusterMassRich, - ClusterMassRichBinArgument, - ClusterMassRichPointArgument, -) - - -@pytest.fixture(name="cluster_mass_rich") -def fixture_cluster_mass_rich() -> ClusterMassRich: - """Initialize cluster object.""" - pivot_redshift = 0.6 - pivot_mass = 14.625862906 - - cluster_mass_rich = ClusterMassRich(pivot_mass, pivot_redshift) - - # Set the parameters to the values used in the test - # they should be such that the variance is always positive. - cluster_mass_rich.mu_p0 = 3.00 - cluster_mass_rich.mu_p1 = 0.086 - cluster_mass_rich.mu_p2 = 0.01 - cluster_mass_rich.sigma_p0 = 3.0 - cluster_mass_rich.sigma_p1 = 0.07 - cluster_mass_rich.sigma_p2 = 0.01 - - return cluster_mass_rich - - -def test_cluster_mass_parameters_function_z(): - for z in np.geomspace(1.0e-18, 2.0, 20): - f_z = ClusterMassRich.cluster_mass_parameters_function( - 0.0, 0.0, (0.0, 0.0, 1.0), 0.0, z - ) - assert f_z == pytest.approx(np.log1p(z), 1.0e-7, 0.0) - - -def test_cluster_mass_parameters_function_M(): - for logM in np.linspace(10.0, 16.0, 20): - f_logM = ClusterMassRich.cluster_mass_parameters_function( - 0.0, 0.0, (0.0, 1.0, 0.0), logM, 0.0 - ) - - assert f_logM == pytest.approx(logM * np.log(10.0), 1.0e-7, 0.0) - - -def test_cluster_mass_lnM_obs_mu_sigma(cluster_mass_rich: ClusterMassRich): - for logM, z in itertools.product( - np.linspace(10.0, 16.0, 20), np.geomspace(1.0e-18, 2.0, 20) - ): - lnM_obs_mu_direct = ClusterMassRich.cluster_mass_parameters_function( - cluster_mass_rich.log_pivot_mass, - cluster_mass_rich.log1p_pivot_redshift, - (cluster_mass_rich.mu_p0, cluster_mass_rich.mu_p1, cluster_mass_rich.mu_p2), - logM, - z, - ) - sigma_direct = ClusterMassRich.cluster_mass_parameters_function( - cluster_mass_rich.log_pivot_mass, - cluster_mass_rich.log1p_pivot_redshift, - ( - cluster_mass_rich.sigma_p0, - cluster_mass_rich.sigma_p1, - cluster_mass_rich.sigma_p2, - ), - logM, - z, - ) - lnM_obs_mu, sigma = cluster_mass_rich.cluster_mass_lnM_obs_mu_sigma(logM, z) - - assert lnM_obs_mu == pytest.approx(lnM_obs_mu_direct, 1.0e-7, 0.0) - assert sigma == pytest.approx(sigma_direct, 1.0e-7, 0.0) - - -def test_cluster_richness_bins_invalid_edges(cluster_mass_rich: ClusterMassRich): - rich_bin_edges = np.array([20]) - - with pytest.raises(ValueError): - _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - - rich_bin_edges = np.array([20, 30, 10]) - with pytest.raises(ValueError): - _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - - rich_bin_edges = np.array([20, 30, 30]) - with pytest.raises(ValueError): - _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - - -def test_cluster_richness_bins(cluster_mass_rich: ClusterMassRich): - rich_bin_edges = np.array([20, 40, 60, 80]) - - richness_bins = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - - for Rl, Ru, richness_bin in zip( - rich_bin_edges[:-1], rich_bin_edges[1:], richness_bins - ): - assert isinstance(richness_bin, ClusterMassRichBinArgument) - assert Rl == richness_bin.logM_obs_lower - assert Ru == richness_bin.logM_obs_upper - - -def test_cluster_bin_probability(cluster_mass_rich: ClusterMassRich): - cluser_mass_rich_bin_argument = ClusterMassRichBinArgument( - cluster_mass_rich, 13.0, 17.0, 1.0, 5.0 - ) - - logM_array = np.linspace(7.0, 26.0, 2000) - for z in np.geomspace(1.0e-18, 2.0, 20): - flip = False - for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): - probability_0 = cluser_mass_rich_bin_argument.p(logM_0, z) - probability_1 = cluser_mass_rich_bin_argument.p(logM_1, z) - - assert probability_0 >= 0 - assert probability_1 >= 0 - - # Probability should be initially monotonically increasing - # and then monotonically decreasing. It should flip only once. - - # Test for the flip - if (not flip) and (probability_1 < probability_0): - flip = True - - # Test for the second flip - if flip and (probability_1 > probability_0): - raise ValueError("Probability flipped twice") - - if flip: - assert probability_1 <= probability_0 - else: - assert probability_1 >= probability_0 - - -def test_cluster_point_probability(cluster_mass_rich: ClusterMassRich): - cluser_mass_rich_bin_argument = ClusterMassRichPointArgument( - cluster_mass_rich, 13.0, 17.0, 2.5 - ) - - logM_array = np.linspace(7.0, 26.0, 2000) - for z in np.geomspace(1.0e-18, 2.0, 20): - flip = False - for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): - probability_0 = cluser_mass_rich_bin_argument.p(logM_0, z) - probability_1 = cluser_mass_rich_bin_argument.p(logM_1, z) - - assert probability_0 >= 0 - assert probability_1 >= 0 - - # Probability density should be initially monotonically increasing - # and then monotonically decreasing. It should flip only once. - - # Test for the flip - if (not flip) and (probability_1 < probability_0): - flip = True - - # Test for the second flip - if flip and (probability_1 > probability_0): - raise ValueError("Probability flipped twice") +# from firecrown.models.cluster_mass_rich_proxy import ( +# ClusterMassRich, +# ClusterMassRichBinArgument, +# ClusterMassRichPointArgument, +# ) + + +# @pytest.fixture(name="cluster_mass_rich") +# def fixture_cluster_mass_rich() -> ClusterMassRich: +# """Initialize cluster object.""" +# pivot_redshift = 0.6 +# pivot_mass = 14.625862906 + +# cluster_mass_rich = ClusterMassRich(pivot_mass, pivot_redshift) + +# # Set the parameters to the values used in the test +# # they should be such that the variance is always positive. +# cluster_mass_rich.mu_p0 = 3.00 +# cluster_mass_rich.mu_p1 = 0.086 +# cluster_mass_rich.mu_p2 = 0.01 +# cluster_mass_rich.sigma_p0 = 3.0 +# cluster_mass_rich.sigma_p1 = 0.07 +# cluster_mass_rich.sigma_p2 = 0.01 + +# return cluster_mass_rich + + +# def test_cluster_mass_parameters_function_z(): +# for z in np.geomspace(1.0e-18, 2.0, 20): +# f_z = ClusterMassRich.cluster_mass_parameters_function( +# 0.0, 0.0, (0.0, 0.0, 1.0), 0.0, z +# ) +# assert f_z == pytest.approx(np.log1p(z), 1.0e-7, 0.0) + + +# def test_cluster_mass_parameters_function_M(): +# for logM in np.linspace(10.0, 16.0, 20): +# f_logM = ClusterMassRich.cluster_mass_parameters_function( +# 0.0, 0.0, (0.0, 1.0, 0.0), logM, 0.0 +# ) + +# assert f_logM == pytest.approx(logM * np.log(10.0), 1.0e-7, 0.0) + + +# def test_cluster_mass_lnM_obs_mu_sigma(cluster_mass_rich: ClusterMassRich): +# for logM, z in itertools.product( +# np.linspace(10.0, 16.0, 20), np.geomspace(1.0e-18, 2.0, 20) +# ): +# lnM_obs_mu_direct = ClusterMassRich.cluster_mass_parameters_function( +# cluster_mass_rich.log_pivot_mass, +# cluster_mass_rich.log1p_pivot_redshift, +# (cluster_mass_rich.mu_p0, cluster_mass_rich.mu_p1, cluster_mass_rich.mu_p2), +# logM, +# z, +# ) +# sigma_direct = ClusterMassRich.cluster_mass_parameters_function( +# cluster_mass_rich.log_pivot_mass, +# cluster_mass_rich.log1p_pivot_redshift, +# ( +# cluster_mass_rich.sigma_p0, +# cluster_mass_rich.sigma_p1, +# cluster_mass_rich.sigma_p2, +# ), +# logM, +# z, +# ) +# lnM_obs_mu, sigma = cluster_mass_rich.cluster_mass_lnM_obs_mu_sigma(logM, z) + +# assert lnM_obs_mu == pytest.approx(lnM_obs_mu_direct, 1.0e-7, 0.0) +# assert sigma == pytest.approx(sigma_direct, 1.0e-7, 0.0) + + +# def test_cluster_richness_bins_invalid_edges(cluster_mass_rich: ClusterMassRich): +# rich_bin_edges = np.array([20]) + +# with pytest.raises(ValueError): +# _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) + +# rich_bin_edges = np.array([20, 30, 10]) +# with pytest.raises(ValueError): +# _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) + +# rich_bin_edges = np.array([20, 30, 30]) +# with pytest.raises(ValueError): +# _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) + + +# def test_cluster_richness_bins(cluster_mass_rich: ClusterMassRich): +# rich_bin_edges = np.array([20, 40, 60, 80]) + +# richness_bins = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) + +# for Rl, Ru, richness_bin in zip( +# rich_bin_edges[:-1], rich_bin_edges[1:], richness_bins +# ): +# assert isinstance(richness_bin, ClusterMassRichBinArgument) +# assert Rl == richness_bin.logM_obs_lower +# assert Ru == richness_bin.logM_obs_upper + + +# def test_cluster_bin_probability(cluster_mass_rich: ClusterMassRich): +# cluser_mass_rich_bin_argument = ClusterMassRichBinArgument( +# cluster_mass_rich, 13.0, 17.0, 1.0, 5.0 +# ) + +# logM_array = np.linspace(7.0, 26.0, 2000) +# for z in np.geomspace(1.0e-18, 2.0, 20): +# flip = False +# for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): +# probability_0 = cluser_mass_rich_bin_argument.p(logM_0, z) +# probability_1 = cluser_mass_rich_bin_argument.p(logM_1, z) + +# assert probability_0 >= 0 +# assert probability_1 >= 0 + +# # Probability should be initially monotonically increasing +# # and then monotonically decreasing. It should flip only once. + +# # Test for the flip +# if (not flip) and (probability_1 < probability_0): +# flip = True + +# # Test for the second flip +# if flip and (probability_1 > probability_0): +# raise ValueError("Probability flipped twice") + +# if flip: +# assert probability_1 <= probability_0 +# else: +# assert probability_1 >= probability_0 + + +# def test_cluster_point_probability(cluster_mass_rich: ClusterMassRich): +# cluser_mass_rich_bin_argument = ClusterMassRichPointArgument( +# cluster_mass_rich, 13.0, 17.0, 2.5 +# ) + +# logM_array = np.linspace(7.0, 26.0, 2000) +# for z in np.geomspace(1.0e-18, 2.0, 20): +# flip = False +# for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): +# probability_0 = cluser_mass_rich_bin_argument.p(logM_0, z) +# probability_1 = cluser_mass_rich_bin_argument.p(logM_1, z) + +# assert probability_0 >= 0 +# assert probability_1 >= 0 + +# # Probability density should be initially monotonically increasing +# # and then monotonically decreasing. It should flip only once. + +# # Test for the flip +# if (not flip) and (probability_1 < probability_0): +# flip = True + +# # Test for the second flip +# if flip and (probability_1 > probability_0): +# raise ValueError("Probability flipped twice") - if flip: - assert probability_1 <= probability_0 - else: - assert probability_1 >= probability_0 +# if flip: +# assert probability_1 <= probability_0 +# else: +# assert probability_1 >= probability_0 From 97b77eeb97e04ecc933fcf291216412b7ec3c6ad Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 12 Oct 2023 12:15:58 -0700 Subject: [PATCH 08/80] Cosmosis running. Number counts seem correct. Need to rethink how we include analytic solutions Currently this is working but not how I want it --- .../cluster_redshift_richness.py | 58 ++++---- firecrown/connector/cosmosis/likelihood.py | 4 +- .../statistic/cluster_number_counts.py | 5 +- firecrown/modeling_tools.py | 6 +- firecrown/models/cluster_abundance.py | 134 +++++++++++------- firecrown/models/kernel.py | 22 ++- firecrown/models/mass_observable.py | 58 ++++---- firecrown/models/redshift.py | 12 +- tests/test_cluster_abundance.py | 12 +- tests/test_cluster_kernels.py | 16 +-- 10 files changed, 175 insertions(+), 152 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 752349dcb..d2775ec30 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -14,22 +14,20 @@ from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster_abundance import ClusterAbundance from firecrown.models.sacc_adapter import SaccAdapter -from firecrown.models.mass_observable import MassRichnessMuSigma, Mass -from firecrown.models.redshift import SpectroscopicRedshift, Redshift +from firecrown.models.mass_observable import MassRichnessMuSigma +from firecrown.models.redshift import SpectroscopicRedshift from firecrown.models.kernel import Completeness, Purity +from firecrown.parameters import ParamsMap -# def build_likelihood(build_parameters): -def build_likelihood(): +def build_likelihood(build_parameters): """ Here we instantiate the number density (or mass function) object. """ # Pull params for the likelihood from build_parameters - # use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) - # use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) - use_cluster_counts = True - use_mean_log_mass = False + use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) + use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) # Read in sacc data sacc_file_nm = "cluster_redshift_richness_sacc_data.fits" @@ -61,46 +59,49 @@ def build_likelihood(): sacc_indices += indices # Finally, build the statistics we want in our likelihood (cluster abundance) + counts = ClusterNumberCounts(use_cluster_counts, use_mean_log_mass) counts.data_vector = DataVector.from_list(data_vector) counts.sacc_indices = np.array(sacc_indices) - counts.sky_area = sacc_adapter.survey_tracer.sky_area - mass_limits = [] - z_limits = [] + mass_proxy_limits = [] + z_proxy_limits = [] if use_cluster_counts: - mass_limits += sacc_adapter.get_mass_tracer_bin_limits( + mass_proxy_limits += sacc_adapter.get_mass_tracer_bin_limits( + sacc_types.cluster_counts + ) + z_proxy_limits += sacc_adapter.get_z_tracer_bin_limits( sacc_types.cluster_counts ) - z_limits += sacc_adapter.get_z_tracer_bin_limits(sacc_types.cluster_counts) if use_mean_log_mass: - mass_limits += sacc_adapter.get_mass_tracer_bin_limits( + mass_proxy_limits += sacc_adapter.get_mass_tracer_bin_limits( sacc_types.cluster_mean_log_mass ) - z_limits += sacc_adapter.get_z_tracer_bin_limits( + z_proxy_limits += sacc_adapter.get_z_tracer_bin_limits( sacc_types.cluster_mean_log_mass ) hmf = ccl.halos.MassFuncTinker08() - cluster_abundance = ClusterAbundance(hmf) + min_mass, max_mass = 13.0, 16.0 + min_z, max_z = 0.1, 2 + sky_area = sacc_adapter.survey_tracer.sky_area + cluster_abundance = ClusterAbundance( + min_mass, max_mass, min_z, max_z, hmf, sky_area + ) # Create and add the kernels you want in your cluster abundance - true_limits = [(0, np.inf)] - mass_kernel = Mass(true_limits) + pivot_mass, pivot_redshift = 14.625862906, 0.6 mass_observable_kernel = MassRichnessMuSigma( - 14.625862906, 0.6, 13.0, 15.0, mass_limits + pivot_mass, pivot_redshift, mass_proxy_limits ) - redshift_kernel = Redshift(true_limits) - redshift_proxy_kernel = SpectroscopicRedshift(z_limits) - completeness_kernel = Completeness() - purity_kernel = Purity() + redshift_proxy_kernel = SpectroscopicRedshift(z_proxy_limits) + # completeness_kernel = Completeness() + # purity_kernel = Purity() - cluster_abundance.add_kernel(mass_kernel) cluster_abundance.add_kernel(mass_observable_kernel) - cluster_abundance.add_kernel(redshift_kernel) cluster_abundance.add_kernel(redshift_proxy_kernel) - cluster_abundance.add_kernel(completeness_kernel) - cluster_abundance.add_kernel(purity_kernel) + # cluster_abundance.add_kernel(completeness_kernel) + # cluster_abundance.add_kernel(purity_kernel) # Put it all together stats_list = [counts] @@ -109,6 +110,3 @@ def build_likelihood(): modeling = ModelingTools(cluster_abundance=cluster_abundance) return lk, modeling - - -build_likelihood() diff --git a/firecrown/connector/cosmosis/likelihood.py b/firecrown/connector/cosmosis/likelihood.py index d9ffa0f98..f356addc3 100644 --- a/firecrown/connector/cosmosis/likelihood.py +++ b/firecrown/connector/cosmosis/likelihood.py @@ -7,7 +7,7 @@ likelihood abstract base class; it the implementation of a CosmoSIS module, not a specific likelihood. """ - +import pdb import cosmosis.datablock from cosmosis.datablock import option_section @@ -116,7 +116,7 @@ def execute(self, sample: cosmosis.datablock) -> int: msg = self.form_error_message(exc) raise RuntimeError(msg) from exc - self.tools.prepare(ccl_cosmo) + self.tools.prepare(ccl_cosmo, firecrown_params) loglike = self.likelihood.compute_loglike(self.tools) derived_params_collection = self.likelihood.get_derived_parameters() diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index d48076b62..439acb5d8 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -4,6 +4,7 @@ from .statistic import Statistic, DataVector, TheoryVector from .source.source import SourceSystematic from ....modeling_tools import ModelingTools +import pdb class ClusterNumberCounts(Statistic): @@ -18,7 +19,6 @@ def __init__( self.theory_vector: Optional[TheoryVector] = None self.use_cluster_counts = cluster_counts self.use_mean_log_mass = mean_log_mass - self.sky_area = 0.0 self.data_vector = DataVector.from_list([]) def read(self, sacc_data: sacc.Sacc): @@ -29,8 +29,6 @@ def get_data_vector(self) -> DataVector: return self.data_vector def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: - tools.cluster_abundance.sky_area = self.sky_area - theory_vector_list = [] cluster_counts_list = [] @@ -38,6 +36,7 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_counts_list = tools.cluster_abundance.compute() theory_vector_list += cluster_counts_list + pdb.set_trace() # if self.use_mean_log_mass: # mean_log_mass_list = [ # self.cluster_abundance.compute_unormalized_mean_logM( diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index 4ddf0bab2..5bded3778 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -7,6 +7,8 @@ from typing import Dict, Optional, final import pyccl.nl_pt + +from firecrown.parameters import ParamsMap from .models.cluster_abundance import ClusterAbundance @@ -55,7 +57,7 @@ def has_pk(self, name: str) -> bool: return False return True - def prepare(self, ccl_cosmo: pyccl.Cosmology) -> None: + def prepare(self, ccl_cosmo: pyccl.Cosmology, params: ParamsMap) -> None: """Prepare the Cosmology for use in likelihoods. This method will prepare the ModelingTools for use in likelihoods. This @@ -74,7 +76,7 @@ def prepare(self, ccl_cosmo: pyccl.Cosmology) -> None: self.pt_calculator.update_ingredients(ccl_cosmo) if self.cluster_abundance is not None: - self.cluster_abundance.update_ingredients(ccl_cosmo) + self.cluster_abundance.update_ingredients(ccl_cosmo, params) @final def reset(self) -> None: diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index 96456d5e3..3f3b6dae9 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -3,12 +3,17 @@ import pyccl.background as bkg import pyccl from scipy.integrate import nquad - -from firecrown.models.kernel import Kernel +from itertools import product +from firecrown.models.kernel import Kernel, KernelType import numpy as np +import pdb +from firecrown.parameters import ParamsMap class ClusterAbundance(object): + _absolute_tolerance = 1e-12 + _relative_tolerance = 1e-4 + @property def sky_area(self) -> float: return self.sky_area_rad * (180.0 / np.pi) ** 2 @@ -23,19 +28,29 @@ def cosmo(self) -> Cosmology: def __init__( self, + min_mass: float, + max_mass: float, + min_z: float, + max_z: float, halo_mass_function: pyccl.halos.MassFunc, - sky_area_rad: float = 4 * np.pi**2, + sky_area: float, ): self.kernels: List[Kernel] = [] self.halo_mass_function = halo_mass_function - self.sky_area_rad = sky_area_rad + self.min_mass = min_mass + self.max_mass = max_mass + self.min_z = min_z + self.max_z = max_z + self.sky_area = sky_area self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel): self.kernels.append(kernel) - def update_ingredients(self, cosmo: Cosmology): + def update_ingredients(self, cosmo: Cosmology, params: ParamsMap): self._cosmo = cosmo + for kernel in self.kernels: + kernel.update(params) def comoving_volume(self, z) -> float: """Differential Comoving Volume at z. @@ -64,64 +79,79 @@ def mass_function(self, mass: float, z: float) -> float: hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) return hmf - def get_abundance_integrand(self): - def integrand(args: List[float], index_lookup: Dict[str, float]): - z = args[index_lookup["z"]] - mass = args[index_lookup["mass"]] - + def get_abundance_integrand(self, bounds_map, args_map): + def integrand(*args): + z = args[bounds_map["z"]] + mass = args[bounds_map["mass"]] integrand = self.comoving_volume(z) * self.mass_function(mass, z) for kernel in self.kernels: - integrand *= kernel.distribution(args, index_lookup) + if kernel.has_analytic_sln: + integrand *= kernel.analytic_solution(args, bounds_map, args_map) + else: + integrand *= kernel.distribution(args, bounds_map) return integrand return integrand - def get_abundance_bounds(self): - index_lookup = {} - bounds_list = [] - # Keep a separate index instead of using enumerate so we don't increment - # when we skip a dirac delta function + def get_integration_bounds(self): + index_lookup = {"mass": 0, "z": 1} + bounds_list = [[(self.min_mass, self.max_mass)], [(self.min_z, self.max_z)]] + idx = 2 + for kernel in self.kernels: + if kernel.integral_bounds is None or kernel.has_analytic_sln: + continue + + if not kernel.is_dirac_delta: + index_lookup[kernel.kernel_type.name] = idx + bounds_list.append(kernel.integral_bounds) + idx += 1 + continue + + # If either z or m has a proxy, and its a dirac delta, just replace the + # limits + if kernel.kernel_type == KernelType.z_proxy: + bounds_list[index_lookup["z"]] = kernel.integral_bounds + elif kernel.kernel_type == KernelType.mass_proxy: + bounds_list[index_lookup["mass"]] = kernel.integral_bounds + + bounds_by_bin = list(product(*bounds_list)) + + return bounds_by_bin, index_lookup + + def get_analytic_args(self): idx = 0 + index_lookup = {} + args = [] for kernel in self.kernels: - if kernel.is_dirac_delta: + if not kernel.has_analytic_sln: continue index_lookup[kernel.kernel_type.name] = idx - bounds_list.append(kernel.integral_bounds) + args.append(kernel.integral_bounds) idx += 1 - return bounds_list, index_lookup + return args, index_lookup def compute(self): - # Args = x0...xn, index_lkp = t0,...tm - # The function to be integrated. - # Has arguments of x0, ... xn, t0, ... tm, where integration is carried out - # over x0, ... xn, which must be floats. Where t0, ... tm are extra arguments - # passed in args. Function signature should be - # func(x0, x1, ..., xn, t0, t1, ..., tm). Integration is carried out in order. - # That is, integration over x0 is the innermost integral, and xn is the - # outermost. - integrand = self.get_abundance_integrand() - bounds_list, index_lookup = self.get_abundance_bounds() - return nquad( - # Function of the form f(x0, x1, ..., xn, t0, t1, ..., tm) - integrand, - # Each element of ranges must be a sequence of 2 numbers - # e.g. ranges[0] = (a,b) for f(x0, x1) - ranges=bounds_list, - # t0,...tn - args=(index_lookup), - opts={"epsabs": 1e-12, "epsrel": 1e-4}, - ) - - # Firecrown specific - def _process_args(self, args): - x = np.array(args[0:-5]) - index_map, arg, ccl_cosmo, mass_arg, redshift_arg = args[-5:] - arg[index_map] = x - redshift_start_index = 2 + redshift_arg.dim - - logM, z = arg[0:2] - proxy_z = arg[2:redshift_start_index] - proxy_m = arg[redshift_start_index:] - - return logM, z, proxy_z, proxy_m, ccl_cosmo, mass_arg, redshift_arg + bounds_by_bin, bounds_map = self.get_integration_bounds() + analytic_args, args_map = self.get_analytic_args() + integrand = self.get_abundance_integrand(bounds_map, args_map) + + cluster_counts = [] + print(bounds_by_bin) + for bounds in bounds_by_bin: + for analytic_sln in analytic_args: + for analytic_bounds in analytic_sln: + cc = nquad( + integrand, + ranges=bounds, + args=analytic_bounds, + opts={ + "epsabs": self._absolute_tolerance, + "epsrel": self._relative_tolerance, + }, + )[0] + print(cc) + cluster_counts.append(cc) + + print(cluster_counts) + return cluster_counts diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py index 2fb5fede3..118bebc46 100644 --- a/firecrown/models/kernel.py +++ b/firecrown/models/kernel.py @@ -16,12 +16,17 @@ class KernelType(Enum): class Kernel(Updatable, ABC): def __init__( - self, kernel_type: KernelType, integral_bounds: List[Tuple[float, float]] = None + self, + kernel_type: KernelType, + is_dirac_delta=False, + integral_bounds: List[Tuple[float, float]] = None, + has_analytic_sln=False, ): super().__init__() self.integral_bounds = integral_bounds - self.is_dirac_delta = integral_bounds is None + self.is_dirac_delta = is_dirac_delta self.kernel_type = kernel_type + self.has_analytic_sln = has_analytic_sln # TODO change name to something that makes more sense for all proxies # Spread? Distribution? @@ -29,10 +34,15 @@ def __init__( def distribution(self, args: List[float], index_lkp: Dict[str, int]): pass + def analytic_solution( + self, args: List[float], index_lkp: Dict[str, int], args_lkp: Dict[str, int] + ): + return + class Completeness(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.completeness, integral_bounds) + def __init__(self): + super().__init__(KernelType.completeness, False, None) def distribution(self, args: List[float], index_lkp: Dict[str, int]): mass = args[index_lkp["mass"]] @@ -49,8 +59,8 @@ def distribution(self, args: List[float], index_lkp: Dict[str, int]): class Purity(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.purity, integral_bounds) + def __init__(self): + super().__init__(KernelType.purity, False, None) def distribution(self, args: List[float], index_lkp: Dict[str, int]): mass_proxy = args[index_lkp["mass_proxy"]] diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py index f75123c8d..a5c839d1d 100644 --- a/firecrown/models/mass_observable.py +++ b/firecrown/models/mass_observable.py @@ -8,6 +8,7 @@ from scipy import special from .. import parameters from .kernel import Kernel, KernelType +import pdb class MassRichnessMuSigma(Kernel): @@ -15,16 +16,12 @@ def __init__( self, pivot_mass, pivot_redshift, - min_mass=13.0, - max_mass=16.0, integral_bounds: List[Tuple[float, float]] = None, ): - super().__init__(KernelType.mass_proxy, integral_bounds) + super().__init__(KernelType.mass_proxy, False, integral_bounds, True) self.pivot_mass = pivot_mass self.pivot_redshift = pivot_redshift - self.pivot_mass = self.pivot_mass * np.log(10.0) - self.min_mass = min_mass - self.max_mass = max_mass + self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) # Updatable parameters @@ -34,44 +31,48 @@ def __init__( self.sigma_p0 = parameters.create() self.sigma_p1 = parameters.create() self.sigma_p2 = parameters.create() + self.limits = self.limits_generator() # Verify this gets called last or first - @staticmethod - def observed_mass( - pivot_mass, log1p_pivot_redshift, p: Tuple[float, float, float], mass, z - ): + def limits_generator(self): + i = 0 + n = len(self.integral_bounds) + + while True: + yield self.integral_bounds[i % n] + i += 1 + + def observed_value(self, p: Tuple[float, float, float], mass, z): """Return observed quantity corrected by redshift and mass.""" ln_mass = mass * np.log(10) - delta_ln_mass = ln_mass - pivot_mass - delta_z = np.log1p(z) - log1p_pivot_redshift + delta_ln_mass = ln_mass - self.pivot_mass + delta_z = np.log1p(z) - self.log1p_pivot_redshift return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - def distribution(self, args: List[float], index_lkp: Dict[str, int]): + def analytic_solution( + self, args: List[float], index_lkp: Dict[str, int], args_lkp: Dict[str, int] + ): mass = args[index_lkp["mass"]] z = args[index_lkp["z"]] - - observed_mean_mass = MassRichnessMuSigma.observed_mass( - self.pivot_mass, - self.log1p_pivot_redshift, + observed_mean_mass = self.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), mass, z, ) - observed_mass_sigma = MassRichnessMuSigma.observed_mass( - self.pivot_mass, - self.log1p_pivot_redshift, + observed_mass_sigma = self.observed_value( (self.sigma_p0, self.sigma_p1, self.sigma_p2), mass, z, ) - - x_min = (observed_mean_mass - self.min_obs_mass * np.log(10.0)) / ( + min_limit = args[2] + max_limit = args[3] + x_min = (observed_mean_mass - min_limit * np.log(10.0)) / ( np.sqrt(2.0) * observed_mass_sigma ) - x_max = (observed_mean_mass - self.max_obs_mass * np.log(10.0)) / ( + x_max = (observed_mean_mass - max_limit * np.log(10.0)) / ( np.sqrt(2.0) * observed_mass_sigma ) @@ -81,6 +82,9 @@ def distribution(self, args: List[float], index_lkp: Dict[str, int]): # pylint: disable-next=no-member return (special.erf(x_min) - special.erf(x_max)) / 2.0 + def distribution(self, args: List[float], index_lkp: Dict[str, int]): + return 0 + # TODO UNDERSTAND THIS # def spread_point(self, logM: float, z: float, *_) -> float: # """Return the probability of the point argument.""" @@ -94,9 +98,11 @@ def distribution(self, args: List[float], index_lkp: Dict[str, int]): # return likelihood * np.log(10.0) -class Mass(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.mass, integral_bounds) +class TrueMass(Kernel): + def __init__( + self, is_dirac_delta=False, integral_bounds: List[Tuple[float, float]] = None + ): + super().__init__(KernelType.mass_proxy, is_dirac_delta, integral_bounds) def distribution(self, args: List[float], index_lkp: Dict[str, int]): return 1.0 diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py index 8d5e47060..cf6600e46 100644 --- a/firecrown/models/redshift.py +++ b/firecrown/models/redshift.py @@ -3,17 +3,9 @@ from typing import List, Tuple, Dict -class Redshift(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.z, integral_bounds) - - def distribution(self, args: List[float], index_lkp: Dict[str, int]): - return 1.0 - - class SpectroscopicRedshift(Kernel): def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.z_proxy, integral_bounds) + super().__init__(KernelType.z_proxy, True, integral_bounds) def distribution(self, args: List[float], index_lkp: Dict[str, int]): return 1.0 @@ -21,7 +13,7 @@ def distribution(self, args: List[float], index_lkp: Dict[str, int]): class DESY1PhotometricRedshift(Kernel): def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.z_proxy, integral_bounds) + super().__init__(KernelType.z_proxy, False) self.sigma_0 = 0.05 def distribution(self, args: List[float], index_lkp: Dict[str, int]): diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 0c97d2478..a0fea34a4 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -57,23 +57,23 @@ def test_cluster_add_kernel(): def test_cluster_abundance_bounds(): hmf = pyccl.halos.MassFuncTinker08() ca = ClusterAbundance(hmf) - bounds, index_lookup = ca.get_abundance_bounds() + bounds, index_lookup = ca.get_integration_bounds() assert len(bounds) == 0 assert len(index_lookup) == 0 ca.add_kernel(MockKernel()) - bounds, index_lookup = ca.get_abundance_bounds() + bounds, index_lookup = ca.get_integration_bounds() assert len(bounds) == 0 assert len(index_lookup) == 0 ca.add_kernel(MockKernel([(0, 1)])) - bounds, index_lookup = ca.get_abundance_bounds() + bounds, index_lookup = ca.get_integration_bounds() assert len(bounds) == 1 assert len(index_lookup) == 1 assert index_lookup["mass"] == 0 assert bounds[index_lookup["mass"]] == [(0, 1)] -def test_cluster_abundance_integrand(): - hmf = pyccl.halos.MassFuncTinker08() - ca = ClusterAbundance(hmf) +# def test_cluster_abundance_integrand(): +# hmf = pyccl.halos.MassFuncTinker08() +# ca = ClusterAbundance(hmf) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index d30ab3e7d..63d24877f 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -1,7 +1,6 @@ from firecrown.models.kernel import Completeness, Purity, KernelType, Kernel -from firecrown.models.mass_observable import Mass, MassRichnessMuSigma +from firecrown.models.mass_observable import TrueMass, MassRichnessMuSigma from firecrown.models.redshift import ( - Redshift, SpectroscopicRedshift, DESY1PhotometricRedshift, ) @@ -33,19 +32,6 @@ def test_spectroscopic_redshift_kernel(): assert srk.integral_bounds[0] == (0, 1) -def test_redshift_kernel(): - rk = Redshift() - assert isinstance(rk, Kernel) - assert rk.kernel_type == KernelType.z - assert rk.is_dirac_delta is True - assert rk.integral_bounds is None - - rk = Redshift([(0, 1)]) - assert rk.is_dirac_delta is False - assert len(rk.integral_bounds) == 1 - assert rk.integral_bounds[0] == (0, 1) - - def test_musigma_kernel(): msk = MassRichnessMuSigma(1, 1) assert isinstance(msk, Kernel) From 87498cfa0545291539d0f7d15303cdb732b7d9e0 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 12 Oct 2023 14:34:40 -0700 Subject: [PATCH 09/80] Changing up the approach to have the cluster module compute by z, m observed bins. --- .../cluster_redshift_richness.py | 59 +------- .../statistic/cluster_number_counts.py | 42 +++++- firecrown/models/cluster_abundance.py | 133 ++++++++++-------- firecrown/models/kernel.py | 33 ++--- firecrown/models/mass_observable.py | 40 ++---- firecrown/models/redshift.py | 12 +- firecrown/models/sacc_adapter.py | 42 +++--- 7 files changed, 166 insertions(+), 195 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index d2775ec30..4fe2f8f31 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -15,7 +15,7 @@ from firecrown.models.cluster_abundance import ClusterAbundance from firecrown.models.sacc_adapter import SaccAdapter from firecrown.models.mass_observable import MassRichnessMuSigma -from firecrown.models.redshift import SpectroscopicRedshift +from firecrown.models.redshift import SpectroscopicRedshift, DESY1PhotometricRedshift from firecrown.models.kernel import Completeness, Purity from firecrown.parameters import ParamsMap @@ -38,63 +38,18 @@ def build_likelihood(build_parameters): sacc_file = os.path.join(sacc_path, sacc_file_nm) sacc_data = sacc.Sacc.load_fits(sacc_file) - sacc_adapter = SaccAdapter( - sacc_data, survey_name, use_cluster_counts, use_mean_log_mass - ) - - # Build the data vector and indices needed for the likelihood - data_vector = [] - sacc_indices = [] - sacc_types = sacc.data_types.standard_types - if use_cluster_counts: - data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) - data_vector += data - sacc_indices += indices - - if use_mean_log_mass: - data, indices = sacc_adapter.get_data_and_indices( - sacc_types.cluster_mean_log_mass - ) - data_vector += data - sacc_indices += indices - - # Finally, build the statistics we want in our likelihood (cluster abundance) - - counts = ClusterNumberCounts(use_cluster_counts, use_mean_log_mass) - counts.data_vector = DataVector.from_list(data_vector) - counts.sacc_indices = np.array(sacc_indices) - - mass_proxy_limits = [] - z_proxy_limits = [] - if use_cluster_counts: - mass_proxy_limits += sacc_adapter.get_mass_tracer_bin_limits( - sacc_types.cluster_counts - ) - z_proxy_limits += sacc_adapter.get_z_tracer_bin_limits( - sacc_types.cluster_counts - ) - if use_mean_log_mass: - mass_proxy_limits += sacc_adapter.get_mass_tracer_bin_limits( - sacc_types.cluster_mean_log_mass - ) - z_proxy_limits += sacc_adapter.get_z_tracer_bin_limits( - sacc_types.cluster_mean_log_mass - ) + counts = ClusterNumberCounts(use_cluster_counts, use_mean_log_mass, survey_name) hmf = ccl.halos.MassFuncTinker08() min_mass, max_mass = 13.0, 16.0 - min_z, max_z = 0.1, 2 - sky_area = sacc_adapter.survey_tracer.sky_area - cluster_abundance = ClusterAbundance( - min_mass, max_mass, min_z, max_z, hmf, sky_area - ) + min_z, max_z = 0.2, 0.8 + cluster_abundance = ClusterAbundance(min_mass, max_mass, min_z, max_z, hmf) # Create and add the kernels you want in your cluster abundance pivot_mass, pivot_redshift = 14.625862906, 0.6 - mass_observable_kernel = MassRichnessMuSigma( - pivot_mass, pivot_redshift, mass_proxy_limits - ) - redshift_proxy_kernel = SpectroscopicRedshift(z_proxy_limits) + mass_observable_kernel = MassRichnessMuSigma(pivot_mass, pivot_redshift) + # redshift_proxy_kernel = SpectroscopicRedshift() + redshift_proxy_kernel = DESY1PhotometricRedshift() # completeness_kernel = Completeness() # purity_kernel = Purity() diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index 439acb5d8..0c60a3a57 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -1,10 +1,13 @@ from __future__ import annotations from typing import List, Optional import sacc + +from firecrown.models.sacc_adapter import SaccAdapter from .statistic import Statistic, DataVector, TheoryVector from .source.source import SourceSystematic from ....modeling_tools import ModelingTools import pdb +import numpy as np class ClusterNumberCounts(Statistic): @@ -12,6 +15,7 @@ def __init__( self, cluster_counts: bool, mean_log_mass: bool, + survey_name: str, systematics: Optional[List[SourceSystematic]] = None, ): super().__init__() @@ -19,9 +23,37 @@ def __init__( self.theory_vector: Optional[TheoryVector] = None self.use_cluster_counts = cluster_counts self.use_mean_log_mass = mean_log_mass + self.survey_name = survey_name self.data_vector = DataVector.from_list([]) def read(self, sacc_data: sacc.Sacc): + # Build the data vector and indices needed for the likelihood + + data_vector = [] + sacc_indices = [] + bin_limits = [] + sacc_types = sacc.data_types.standard_types + sacc_adapter = SaccAdapter( + sacc_data, self.survey_name, self.use_cluster_counts, self.use_mean_log_mass + ) + + if self.use_cluster_counts: + data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) + bin_limits += sacc_adapter.get_bin_limits(sacc_types.cluster_counts) + data_vector += data + sacc_indices += indices + + if self.use_mean_log_mass: + data, indices = sacc_adapter.get_data_and_indices( + sacc_types.cluster_mean_log_mass + ) + bin_limits += sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) + data_vector += data + sacc_indices += indices + self.sky_area = sacc_adapter.survey_tracer.sky_area + self.bin_limits = bin_limits + self.data_vector = DataVector.from_list(data_vector) + self.sacc_indices = np.array(sacc_indices) super().read(sacc_data) def get_data_vector(self) -> DataVector: @@ -29,14 +61,20 @@ def get_data_vector(self) -> DataVector: return self.data_vector def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: + tools.cluster_abundance.sky_area = self.sky_area + theory_vector_list = [] cluster_counts_list = [] if self.use_cluster_counts: - cluster_counts_list = tools.cluster_abundance.compute() + for z_proxy_limits, mass_proxy_limits in self.bin_limits: + cluster_counts = tools.cluster_abundance.compute( + z_proxy_limits, mass_proxy_limits + ) + cluster_counts_list.append(cluster_counts) theory_vector_list += cluster_counts_list + print(theory_vector_list) - pdb.set_trace() # if self.use_mean_log_mass: # mean_log_mass_list = [ # self.cluster_abundance.compute_unormalized_mean_logM( diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index 3f3b6dae9..076a261a0 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -33,7 +33,7 @@ def __init__( min_z: float, max_z: float, halo_mass_function: pyccl.halos.MassFunc, - sky_area: float, + sky_area: float = 0, ): self.kernels: List[Kernel] = [] self.halo_mass_function = halo_mass_function @@ -79,79 +79,88 @@ def mass_function(self, mass: float, z: float) -> float: hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) return hmf - def get_abundance_integrand(self, bounds_map, args_map): + def get_abundance_integrand(self, bounds_map): + # Use the bounds mapping from the outer scope. + def integrand(*args): - z = args[bounds_map["z"]] - mass = args[bounds_map["mass"]] + z = args[bounds_map[KernelType.z.name]] + mass = args[bounds_map[KernelType.mass.name]] + integrand = self.comoving_volume(z) * self.mass_function(mass, z) for kernel in self.kernels: - if kernel.has_analytic_sln: - integrand *= kernel.analytic_solution(args, bounds_map, args_map) - else: - integrand *= kernel.distribution(args, bounds_map) + integrand *= ( + kernel.analytic_solution(args, bounds_map) + if kernel.has_analytic_sln + else kernel.distribution(args, bounds_map) + ) return integrand return integrand - def get_integration_bounds(self): - index_lookup = {"mass": 0, "z": 1} - bounds_list = [[(self.min_mass, self.max_mass)], [(self.min_z, self.max_z)]] - idx = 2 - for kernel in self.kernels: - if kernel.integral_bounds is None or kernel.has_analytic_sln: - continue + def get_analytic_kernels(self): + return [x for x in self.kernels if x.has_analytic_sln] + + def get_dirac_delta_kernels(self): + return [x for x in self.kernels if x.is_dirac_delta] - if not kernel.is_dirac_delta: - index_lookup[kernel.kernel_type.name] = idx + def get_integrable_kernels(self): + return [ + x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln + ] + + def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): + bounds_map = {KernelType.mass.name: 0, KernelType.z.name: 1} + bounds_list = [(self.min_mass, self.max_mass), (self.min_z, self.max_z)] + start_idx = len(bounds_map.keys()) + + for kernel in self.get_dirac_delta_kernels(): + # If any kernel is a dirac delta for z or M, just replace the + # True limits with the proxy limits + if kernel.kernel_type == KernelType.z_proxy: + bounds_list[bounds_map[KernelType.z.name]] = z_proxy_limits + elif kernel.kernel_type == KernelType.mass_proxy: + bounds_list[bounds_map[KernelType.mass.name]] = mass_proxy_limits + + for kernel in self.get_integrable_kernels(): + # If any kernel is not a dirac delta, integrate over the relevant limits + if kernel.kernel_type == KernelType.z_proxy: + bounds_map[kernel.kernel_type.name] = start_idx + bounds_list.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + bounds_map[kernel.kernel_type.name] = start_idx + bounds_list.append(mass_proxy_limits) + else: + bounds_map[kernel.kernel_type.name] = start_idx bounds_list.append(kernel.integral_bounds) - idx += 1 - continue + start_idx += 1 - # If either z or m has a proxy, and its a dirac delta, just replace the - # limits + extra_args = [] + for kernel in self.get_analytic_kernels(): + # Lastly, don't integrate any analyticly solved kernels, just solve them. if kernel.kernel_type == KernelType.z_proxy: - bounds_list[index_lookup["z"]] = kernel.integral_bounds + bounds_map[kernel.kernel_type.name] = start_idx + extra_args.append(z_proxy_limits) elif kernel.kernel_type == KernelType.mass_proxy: - bounds_list[index_lookup["mass"]] = kernel.integral_bounds + bounds_map[kernel.kernel_type.name] = start_idx + extra_args.append(mass_proxy_limits) - bounds_by_bin = list(product(*bounds_list)) + start_idx += 1 - return bounds_by_bin, index_lookup + return bounds_list, extra_args, bounds_map - def get_analytic_args(self): - idx = 0 - index_lookup = {} - args = [] - for kernel in self.kernels: - if not kernel.has_analytic_sln: - continue - index_lookup[kernel.kernel_type.name] = idx - args.append(kernel.integral_bounds) - idx += 1 - - return args, index_lookup - - def compute(self): - bounds_by_bin, bounds_map = self.get_integration_bounds() - analytic_args, args_map = self.get_analytic_args() - integrand = self.get_abundance_integrand(bounds_map, args_map) - - cluster_counts = [] - print(bounds_by_bin) - for bounds in bounds_by_bin: - for analytic_sln in analytic_args: - for analytic_bounds in analytic_sln: - cc = nquad( - integrand, - ranges=bounds, - args=analytic_bounds, - opts={ - "epsabs": self._absolute_tolerance, - "epsrel": self._relative_tolerance, - }, - )[0] - print(cc) - cluster_counts.append(cc) - - print(cluster_counts) - return cluster_counts + def compute(self, z_proxy_limits, mass_proxy_limits): + bounds, extra_args, bounds_map = self.get_integration_bounds( + z_proxy_limits, mass_proxy_limits + ) + integrand = self.get_abundance_integrand(bounds_map) + cc = nquad( + integrand, + ranges=bounds, + args=extra_args, + opts={ + "epsabs": self._absolute_tolerance, + "epsrel": self._relative_tolerance, + }, + )[0] + print(bounds, cc) + return cc diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py index 118bebc46..034d3085b 100644 --- a/firecrown/models/kernel.py +++ b/firecrown/models/kernel.py @@ -1,7 +1,7 @@ from typing import List, Tuple, Dict import numpy as np from firecrown.updatable import Updatable -from abc import ABC, abstractmethod +from abc import ABC from enum import Enum @@ -19,8 +19,8 @@ def __init__( self, kernel_type: KernelType, is_dirac_delta=False, - integral_bounds: List[Tuple[float, float]] = None, has_analytic_sln=False, + integral_bounds: List[Tuple[float, float]] = None, ): super().__init__() self.integral_bounds = integral_bounds @@ -28,25 +28,20 @@ def __init__( self.kernel_type = kernel_type self.has_analytic_sln = has_analytic_sln - # TODO change name to something that makes more sense for all proxies - # Spread? Distribution? - @abstractmethod - def distribution(self, args: List[float], index_lkp: Dict[str, int]): - pass + def distribution(self, args: List[float], args_map: Dict[str, int]): + raise NotImplementedError() - def analytic_solution( - self, args: List[float], index_lkp: Dict[str, int], args_lkp: Dict[str, int] - ): - return + def analytic_solution(self, args: List[float], args_map: Dict[str, int]): + raise NotImplementedError() class Completeness(Kernel): def __init__(self): - super().__init__(KernelType.completeness, False, None) + super().__init__(KernelType.completeness) - def distribution(self, args: List[float], index_lkp: Dict[str, int]): - mass = args[index_lkp["mass"]] - z = args[index_lkp["z"]] + def distribution(self, args: List[float], args_map: Dict[str, int]): + mass = args[args_map[KernelType.mass.name]] + z = args[args_map[KernelType.z.name]] # TODO improve parameter names a_nc = 1.1321 b_nc = 0.7751 @@ -60,12 +55,12 @@ def distribution(self, args: List[float], index_lkp: Dict[str, int]): class Purity(Kernel): def __init__(self): - super().__init__(KernelType.purity, False, None) + super().__init__(KernelType.purity) def distribution(self, args: List[float], index_lkp: Dict[str, int]): - mass_proxy = args[index_lkp["mass_proxy"]] - z = args[index_lkp["z"]] - # TODO improve parameter names + mass_proxy = args[index_lkp[KernelType.mass_proxy.name]] + z = args[index_lkp[KernelType.z.name]] + ln_r = np.log(10**mass_proxy) a_nc = np.log(10) * 0.8612 b_nc = np.log(10) * 0.3527 diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py index a5c839d1d..c5edd55e8 100644 --- a/firecrown/models/mass_observable.py +++ b/firecrown/models/mass_observable.py @@ -8,7 +8,6 @@ from scipy import special from .. import parameters from .kernel import Kernel, KernelType -import pdb class MassRichnessMuSigma(Kernel): @@ -18,7 +17,7 @@ def __init__( pivot_redshift, integral_bounds: List[Tuple[float, float]] = None, ): - super().__init__(KernelType.mass_proxy, False, integral_bounds, True) + super().__init__(KernelType.mass_proxy, False, True, integral_bounds) self.pivot_mass = pivot_mass self.pivot_redshift = pivot_redshift self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) @@ -31,18 +30,9 @@ def __init__( self.sigma_p0 = parameters.create() self.sigma_p1 = parameters.create() self.sigma_p2 = parameters.create() - self.limits = self.limits_generator() # Verify this gets called last or first - def limits_generator(self): - i = 0 - n = len(self.integral_bounds) - - while True: - yield self.integral_bounds[i % n] - i += 1 - def observed_value(self, p: Tuple[float, float, float], mass, z): """Return observed quantity corrected by redshift and mass.""" @@ -52,11 +42,11 @@ def observed_value(self, p: Tuple[float, float, float], mass, z): return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - def analytic_solution( - self, args: List[float], index_lkp: Dict[str, int], args_lkp: Dict[str, int] - ): - mass = args[index_lkp["mass"]] - z = args[index_lkp["z"]] + def analytic_solution(self, args: List[float], index_lkp: Dict[str, int]): + mass = args[index_lkp[KernelType.mass.name]] + z = args[index_lkp[KernelType.z.name]] + mass_limits = args[index_lkp[self.kernel_type.name]] + observed_mean_mass = self.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), mass, @@ -67,24 +57,18 @@ def analytic_solution( mass, z, ) - min_limit = args[2] - max_limit = args[3] - x_min = (observed_mean_mass - min_limit * np.log(10.0)) / ( + + x_min = (observed_mean_mass - mass_limits[0] * np.log(10.0)) / ( np.sqrt(2.0) * observed_mass_sigma ) - x_max = (observed_mean_mass - max_limit * np.log(10.0)) / ( + x_max = (observed_mean_mass - mass_limits[1] * np.log(10.0)) / ( np.sqrt(2.0) * observed_mass_sigma ) if x_max > 3.0 or x_min < -3.0: - # pylint: disable-next=no-member return -(special.erfc(x_min) - special.erfc(x_max)) / 2.0 - # pylint: disable-next=no-member return (special.erf(x_min) - special.erf(x_max)) / 2.0 - def distribution(self, args: List[float], index_lkp: Dict[str, int]): - return 0 - # TODO UNDERSTAND THIS # def spread_point(self, logM: float, z: float, *_) -> float: # """Return the probability of the point argument.""" @@ -99,10 +83,8 @@ def distribution(self, args: List[float], index_lkp: Dict[str, int]): class TrueMass(Kernel): - def __init__( - self, is_dirac_delta=False, integral_bounds: List[Tuple[float, float]] = None - ): - super().__init__(KernelType.mass_proxy, is_dirac_delta, integral_bounds) + def __init__(self): + super().__init__(KernelType.mass_proxy, True) def distribution(self, args: List[float], index_lkp: Dict[str, int]): return 1.0 diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py index cf6600e46..1d2e28155 100644 --- a/firecrown/models/redshift.py +++ b/firecrown/models/redshift.py @@ -4,21 +4,21 @@ class SpectroscopicRedshift(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.z_proxy, True, integral_bounds) + def __init__(self): + super().__init__(KernelType.z_proxy, True) def distribution(self, args: List[float], index_lkp: Dict[str, int]): return 1.0 class DESY1PhotometricRedshift(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.z_proxy, False) + def __init__(self): + super().__init__(KernelType.z_proxy) self.sigma_0 = 0.05 def distribution(self, args: List[float], index_lkp: Dict[str, int]): - z_proxy = args[index_lkp["z_proxy"]] - z = args[index_lkp["z"]] + z_proxy = args[index_lkp[KernelType.z_proxy.name]] + z = args[index_lkp[KernelType.z.name]] sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) diff --git a/firecrown/models/sacc_adapter.py b/firecrown/models/sacc_adapter.py index 2277311ca..f74e99d3a 100644 --- a/firecrown/models/sacc_adapter.py +++ b/firecrown/models/sacc_adapter.py @@ -25,23 +25,22 @@ def __init__( if not isinstance(self.survey_tracer, SurveyTracer): raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") - def filter_tracers(self, data_type): + def get_filtered_tracers(self, data_type): all_tracers = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) self.validate_tracers(all_tracers, data_type) - self.survey_mask = all_tracers[:, self._survey_index] == self.survey_nm - my_tracers = all_tracers[self.survey_mask] - self.z_tracers = np.unique(my_tracers[:, self._redshift_index]) - self.mass_tracers = np.unique(my_tracers[:, self._mass_index]) + survey_mask = all_tracers[:, self._survey_index] == self.survey_nm + filtered_tracers = all_tracers[survey_mask] + return filtered_tracers, survey_mask def get_data_and_indices(self, data_type): - self.filter_tracers(data_type) + _, survey_mask = self.get_filtered_tracers(data_type) data_vector_list = list( - self.sacc_data.get_mean(data_type=data_type)[self.survey_mask] + self.sacc_data.get_mean(data_type=data_type)[survey_mask] ) sacc_indices_list = list( - self.sacc_data.indices(data_type=data_type)[self.survey_mask] + self.sacc_data.indices(data_type=data_type)[survey_mask] ) return data_vector_list, sacc_indices_list @@ -59,22 +58,15 @@ def validate_tracers(self, tracers_combinations, data_type): "redshift argument and mass argument tracers." ) - def get_mass_tracer_bin_limits(self, data_type): - self.filter_tracers(data_type) + def get_bin_limits(self, data_type): + filtered_tracers, _ = self.get_filtered_tracers(data_type) - mass_bounds = [ - (self.sacc_data.get_tracer(x).lower, self.sacc_data.get_tracer(x).upper) - for x in self.mass_tracers - ] - - return mass_bounds - - def get_z_tracer_bin_limits(self, data_type): - self.filter_tracers(data_type) - - z_bounds = [ - (self.sacc_data.get_tracer(x).lower, self.sacc_data.get_tracer(x).upper) - for x in self.z_tracers - ] + tracers = [] + for _, z_tracer, mass_tracer in filtered_tracers: + z_data = self.sacc_data.get_tracer(z_tracer) + mass_data = self.sacc_data.get_tracer(mass_tracer) + tracers.append( + [(z_data.lower, z_data.upper), (mass_data.lower, mass_data.upper)] + ) - return z_bounds + return tracers From 9826c893e18384cbaf39105703213181eb4c5589 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 13 Oct 2023 09:44:47 -0700 Subject: [PATCH 10/80] Changes to use numcosmo integration instead. --- .../cluster_redshift_richness.py | 4 +- .../statistic/cluster_number_counts.py | 9 +- firecrown/models/cluster_abundance.py | 111 ++++++++++++++---- firecrown/models/kernel.py | 23 ++-- firecrown/models/mass_observable.py | 17 ++- firecrown/models/redshift.py | 6 +- 6 files changed, 125 insertions(+), 45 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 4fe2f8f31..ccfafb13c 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -48,8 +48,8 @@ def build_likelihood(build_parameters): # Create and add the kernels you want in your cluster abundance pivot_mass, pivot_redshift = 14.625862906, 0.6 mass_observable_kernel = MassRichnessMuSigma(pivot_mass, pivot_redshift) - # redshift_proxy_kernel = SpectroscopicRedshift() - redshift_proxy_kernel = DESY1PhotometricRedshift() + redshift_proxy_kernel = SpectroscopicRedshift() + # redshift_proxy_kernel = DESY1PhotometricRedshift() # completeness_kernel = Completeness() # purity_kernel = Purity() diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index 0c60a3a57..7b00c7236 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -8,6 +8,8 @@ from ....modeling_tools import ModelingTools import pdb import numpy as np +import cProfile +from pstats import SortKey class ClusterNumberCounts(Statistic): @@ -18,6 +20,7 @@ def __init__( survey_name: str, systematics: Optional[List[SourceSystematic]] = None, ): + self.pr = cProfile.Profile() super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None @@ -50,6 +53,7 @@ def read(self, sacc_data: sacc.Sacc): bin_limits += sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) data_vector += data sacc_indices += indices + self.sky_area = sacc_adapter.survey_tracer.sky_area self.bin_limits = bin_limits self.data_vector = DataVector.from_list(data_vector) @@ -61,6 +65,7 @@ def get_data_vector(self) -> DataVector: return self.data_vector def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: + # self.pr.enable() tools.cluster_abundance.sky_area = self.sky_area theory_vector_list = [] @@ -73,7 +78,6 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: ) cluster_counts_list.append(cluster_counts) theory_vector_list += cluster_counts_list - print(theory_vector_list) # if self.use_mean_log_mass: # mean_log_mass_list = [ @@ -86,4 +90,7 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: # ) # ] # theory_vector_list += mean_log_mass_list + # self.pr.disable() + # self.pr.dump_stats("profile.prof") + return TheoryVector.from_list(theory_vector_list) diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster_abundance.py index 076a261a0..501c96626 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster_abundance.py @@ -1,12 +1,12 @@ -from typing import List, Dict +from typing import List, Tuple from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl from scipy.integrate import nquad -from itertools import product -from firecrown.models.kernel import Kernel, KernelType +from firecrown.models.kernel import Kernel, KernelType, ArgsMapper import numpy as np import pdb +from numcosmo_py import Ncm from firecrown.parameters import ParamsMap @@ -66,9 +66,9 @@ def comoving_volume(self, z) -> float: h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) dV = ( - ((1.0 + z) ** 2) + pyccl.physical_constants.CLIGHT_HMPC + * ((1.0 + z) ** 2) * (angular_diam_dist**2) - * pyccl.physical_constants.CLIGHT_HMPC / self.cosmo["h"] / h_over_h0 ) @@ -79,19 +79,20 @@ def mass_function(self, mass: float, z: float) -> float: hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) return hmf - def get_abundance_integrand(self, bounds_map): + def get_abundance_integrand(self): # Use the bounds mapping from the outer scope. - def integrand(*args): - z = args[bounds_map[KernelType.z.name]] - mass = args[bounds_map[KernelType.mass.name]] + def integrand(*int_args): + args_map = int_args[-1] + z = int_args[0][args_map.integral_bounds[KernelType.z.name]] + mass = int_args[0][args_map.integral_bounds[KernelType.mass.name]] integrand = self.comoving_volume(z) * self.mass_function(mass, z) for kernel in self.kernels: integrand *= ( - kernel.analytic_solution(args, bounds_map) + kernel.analytic_solution(int_args, args_map) if kernel.has_analytic_sln - else kernel.distribution(args, bounds_map) + else kernel.distribution(int_args, args_map) ) return integrand @@ -109,58 +110,118 @@ def get_integrable_kernels(self): ] def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): - bounds_map = {KernelType.mass.name: 0, KernelType.z.name: 1} + args_mapping = ArgsMapper() + args_mapping.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} bounds_list = [(self.min_mass, self.max_mass), (self.min_z, self.max_z)] - start_idx = len(bounds_map.keys()) + + start_idx = len(args_mapping.integral_bounds.keys()) for kernel in self.get_dirac_delta_kernels(): # If any kernel is a dirac delta for z or M, just replace the # True limits with the proxy limits if kernel.kernel_type == KernelType.z_proxy: - bounds_list[bounds_map[KernelType.z.name]] = z_proxy_limits + bounds_list[1] = z_proxy_limits elif kernel.kernel_type == KernelType.mass_proxy: - bounds_list[bounds_map[KernelType.mass.name]] = mass_proxy_limits + bounds_list[0] = mass_proxy_limits for kernel in self.get_integrable_kernels(): # If any kernel is not a dirac delta, integrate over the relevant limits if kernel.kernel_type == KernelType.z_proxy: - bounds_map[kernel.kernel_type.name] = start_idx + args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx bounds_list.append(z_proxy_limits) elif kernel.kernel_type == KernelType.mass_proxy: - bounds_map[kernel.kernel_type.name] = start_idx + args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx bounds_list.append(mass_proxy_limits) else: - bounds_map[kernel.kernel_type.name] = start_idx + args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx bounds_list.append(kernel.integral_bounds) start_idx += 1 extra_args = [] + start_idx = 0 for kernel in self.get_analytic_kernels(): # Lastly, don't integrate any analyticly solved kernels, just solve them. if kernel.kernel_type == KernelType.z_proxy: - bounds_map[kernel.kernel_type.name] = start_idx + args_mapping.extra_args[kernel.kernel_type.name] = start_idx extra_args.append(z_proxy_limits) elif kernel.kernel_type == KernelType.mass_proxy: - bounds_map[kernel.kernel_type.name] = start_idx + args_mapping.extra_args[kernel.kernel_type.name] = start_idx extra_args.append(mass_proxy_limits) start_idx += 1 - return bounds_list, extra_args, bounds_map + return bounds_list, extra_args, args_mapping + + def get_argument_mapper(self): + return def compute(self, z_proxy_limits, mass_proxy_limits): - bounds, extra_args, bounds_map = self.get_integration_bounds( + bounds, extra_args, args_mapping = self.get_integration_bounds( z_proxy_limits, mass_proxy_limits ) - integrand = self.get_abundance_integrand(bounds_map) + + integrand = self.get_abundance_integrand() + # cc = self._scipy_nquad_integrate(integrand, bounds, args_mapping, extra_args) + + cc = self._numcosmo_integrate(integrand, bounds, args_mapping, extra_args) + return cc + + def _numcosmo_integrate(self, integrand, bounds, bounds_map, extra_args): + Ncm.cfg_init() + int_nd = CountsIntegralND( + len(bounds), + integrand, + extra_args, + bounds_map, + ) + int_nd.set_method(Ncm.IntegralNDMethod.P_V) + int_nd.set_reltol(self._relative_tolerance) + int_nd.set_abstol(self._absolute_tolerance) + res = Ncm.Vector.new(1) + err = Ncm.Vector.new(1) + + bl, bu = zip(*bounds) + int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) + return res.get(0) + + def _scipy_nquad_integrate(self, integrand, bounds, bounds_map, extra_args): cc = nquad( integrand, ranges=bounds, - args=extra_args, + args=(extra_args, bounds_map), opts={ "epsabs": self._absolute_tolerance, "epsrel": self._relative_tolerance, }, )[0] - print(bounds, cc) + return cc + + +class CountsIntegralND(Ncm.IntegralND): + """Integral subclass used by the ClusterAbundance + to compute the integrals using numcosmo.""" + + def __init__(self, dim, fun, *args): + super().__init__() + self.dim = dim + self.fun = fun + self.args = args + + # pylint: disable-next=arguments-differ + def do_get_dimensions(self) -> Tuple[int, int]: + """Get number of dimensions.""" + return self.dim, 1 + + # pylint: disable-next=arguments-differ + def do_integrand( + self, + x_vec: Ncm.Vector, + dim: int, + npoints: int, + _fdim: int, + fval_vec: Ncm.Vector, + ) -> None: + """Integrand function.""" + x = np.array(x_vec.dup_array()).reshape(npoints, dim) + fval_vec.set_array([self.fun(x_i, *self.args) for x_i in x]) diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py index 034d3085b..b9af85bb1 100644 --- a/firecrown/models/kernel.py +++ b/firecrown/models/kernel.py @@ -3,6 +3,13 @@ from firecrown.updatable import Updatable from abc import ABC from enum import Enum +from dataclasses import dataclass, field + + +@dataclass +class ArgsMapper: + integral_bounds: dict = field(default_factory=lambda: {}) + extra_args: dict = field(default_factory=lambda: {}) class KernelType(Enum): @@ -28,10 +35,10 @@ def __init__( self.kernel_type = kernel_type self.has_analytic_sln = has_analytic_sln - def distribution(self, args: List[float], args_map: Dict[str, int]): + def distribution(self, args: List[float], args_map: ArgsMapper): raise NotImplementedError() - def analytic_solution(self, args: List[float], args_map: Dict[str, int]): + def analytic_solution(self, args: List[float], args_map: ArgsMapper): raise NotImplementedError() @@ -39,9 +46,9 @@ class Completeness(Kernel): def __init__(self): super().__init__(KernelType.completeness) - def distribution(self, args: List[float], args_map: Dict[str, int]): - mass = args[args_map[KernelType.mass.name]] - z = args[args_map[KernelType.z.name]] + def distribution(self, args: List[float], args_map: ArgsMapper): + mass = args[args_map.integral_bounds[KernelType.mass.name]] + z = args[args_map.integral_bounds[KernelType.z.name]] # TODO improve parameter names a_nc = 1.1321 b_nc = 0.7751 @@ -57,9 +64,9 @@ class Purity(Kernel): def __init__(self): super().__init__(KernelType.purity) - def distribution(self, args: List[float], index_lkp: Dict[str, int]): - mass_proxy = args[index_lkp[KernelType.mass_proxy.name]] - z = args[index_lkp[KernelType.z.name]] + def distribution(self, args: List[float], index_lkp: ArgsMapper): + mass_proxy = args[index_lkp.integral_bounds[KernelType.mass_proxy.name]] + z = args[index_lkp.integral_bounds[KernelType.z.name]] ln_r = np.log(10**mass_proxy) a_nc = np.log(10) * 0.8612 diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py index c5edd55e8..6fcbc6165 100644 --- a/firecrown/models/mass_observable.py +++ b/firecrown/models/mass_observable.py @@ -6,8 +6,10 @@ import numpy as np from scipy import special + from .. import parameters -from .kernel import Kernel, KernelType +from .kernel import Kernel, KernelType, ArgsMapper +import pdb class MassRichnessMuSigma(Kernel): @@ -42,10 +44,13 @@ def observed_value(self, p: Tuple[float, float, float], mass, z): return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - def analytic_solution(self, args: List[float], index_lkp: Dict[str, int]): - mass = args[index_lkp[KernelType.mass.name]] - z = args[index_lkp[KernelType.z.name]] - mass_limits = args[index_lkp[self.kernel_type.name]] + def analytic_solution(self, args: List[float], index_lkp: ArgsMapper): + # pdb.set_trace() + bounds = args[0] + extra_args = args[1] + mass = bounds[index_lkp.integral_bounds[KernelType.mass.name]] + z = bounds[index_lkp.integral_bounds[KernelType.z.name]] + mass_limits = extra_args[index_lkp.extra_args[self.kernel_type.name]] observed_mean_mass = self.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), @@ -86,5 +91,5 @@ class TrueMass(Kernel): def __init__(self): super().__init__(KernelType.mass_proxy, True) - def distribution(self, args: List[float], index_lkp: Dict[str, int]): + def distribution(self, args: List[float], index_lkp: ArgsMapper): return 1.0 diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py index 1d2e28155..b88e80992 100644 --- a/firecrown/models/redshift.py +++ b/firecrown/models/redshift.py @@ -1,5 +1,5 @@ import numpy as np -from firecrown.models.kernel import Kernel, KernelType +from firecrown.models.kernel import Kernel, KernelType, ArgsMapper from typing import List, Tuple, Dict @@ -7,7 +7,7 @@ class SpectroscopicRedshift(Kernel): def __init__(self): super().__init__(KernelType.z_proxy, True) - def distribution(self, args: List[float], index_lkp: Dict[str, int]): + def distribution(self, args: List[float], index_lkp: ArgsMapper): return 1.0 @@ -16,7 +16,7 @@ def __init__(self): super().__init__(KernelType.z_proxy) self.sigma_0 = 0.05 - def distribution(self, args: List[float], index_lkp: Dict[str, int]): + def distribution(self, args: List[float], index_lkp: ArgsMapper): z_proxy = args[index_lkp[KernelType.z_proxy.name]] z = args[index_lkp[KernelType.z.name]] From 81ccacddc6749a3b769fdf4d0dfa9c0957509597 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 13 Oct 2023 10:52:34 -0700 Subject: [PATCH 11/80] Moving files around to directories that make more sense. This is going to show up as a delete and add in git. --- .../cluster_redshift_richness.py | 88 ++++---- firecrown/integrator/__init__.py | 1 + firecrown/integrator/integrator.py | 86 ++++++++ .../statistic/cluster_number_counts.py | 10 +- firecrown/modeling_tools.py | 2 +- firecrown/models/cluster/__init__.py | 0 .../abundance.py} | 104 ++------- firecrown/models/cluster/kernel.py | 199 ++++++++++++++++++ firecrown/models/kernel.py | 83 -------- firecrown/models/mass_observable.py | 95 --------- firecrown/models/redshift.py | 26 --- 11 files changed, 360 insertions(+), 334 deletions(-) create mode 100644 firecrown/integrator/__init__.py create mode 100644 firecrown/integrator/integrator.py create mode 100644 firecrown/models/cluster/__init__.py rename firecrown/models/{cluster_abundance.py => cluster/abundance.py} (64%) create mode 100644 firecrown/models/cluster/kernel.py delete mode 100644 firecrown/models/kernel.py delete mode 100644 firecrown/models/mass_observable.py delete mode 100644 firecrown/models/redshift.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index ccfafb13c..471741c84 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -1,23 +1,52 @@ """Likelihood factory function for cluster number counts.""" import os -import numpy as np import pyccl as ccl import sacc +from firecrown.integrator.integrator import NumCosmoIntegrator +from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( ClusterNumberCounts, - DataVector, ) -from firecrown.likelihood.gauss_family.gaussian import ConstGaussian -from firecrown.modeling_tools import ModelingTools -from firecrown.models.cluster_abundance import ClusterAbundance from firecrown.models.sacc_adapter import SaccAdapter -from firecrown.models.mass_observable import MassRichnessMuSigma -from firecrown.models.redshift import SpectroscopicRedshift, DESY1PhotometricRedshift -from firecrown.models.kernel import Completeness, Purity -from firecrown.parameters import ParamsMap +from firecrown.modeling_tools import ModelingTools +from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.kernel import ( + Completeness, + DESY1PhotometricRedshift, + MassRichnessMuSigma, + Purity, + SpectroscopicRedshift, +) + + +def get_cluster_abundance(sky_area): + integrator = NumCosmoIntegrator() + hmf = ccl.halos.MassFuncTinker08() + min_mass, max_mass = 13.0, 16.0 + min_z, max_z = 0.2, 0.8 + cluster_abundance = ClusterAbundance( + min_mass, max_mass, min_z, max_z, hmf, sky_area, integrator + ) + + # Create and add the kernels you want in your cluster abundance + pivot_mass, pivot_redshift = 14.625862906, 0.6 + mass_observable_kernel = MassRichnessMuSigma(pivot_mass, pivot_redshift) + cluster_abundance.add_kernel(mass_observable_kernel) + + redshift_proxy_kernel = SpectroscopicRedshift() + # redshift_proxy_kernel = DESY1PhotometricRedshift() + cluster_abundance.add_kernel(redshift_proxy_kernel) + + # completeness_kernel = Completeness() + # cluster_abundance.add_kernel(completeness_kernel) + + # # purity_kernel = Purity() + # cluster_abundance.add_kernel(purity_kernel) + + return cluster_abundance def build_likelihood(build_parameters): @@ -28,40 +57,23 @@ def build_likelihood(build_parameters): # Pull params for the likelihood from build_parameters use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) + survey_name = "numcosmo_simulated_redshift_richness" + likelihood = ConstGaussian( + [ClusterNumberCounts(use_cluster_counts, use_mean_log_mass, survey_name)] + ) # Read in sacc data sacc_file_nm = "cluster_redshift_richness_sacc_data.fits" - survey_name = "numcosmo_simulated_redshift_richness" sacc_path = os.path.expanduser( os.path.expandvars("${FIRECROWN_DIR}/examples/cluster_number_counts/") ) - sacc_file = os.path.join(sacc_path, sacc_file_nm) - - sacc_data = sacc.Sacc.load_fits(sacc_file) - counts = ClusterNumberCounts(use_cluster_counts, use_mean_log_mass, survey_name) + sacc_data = sacc.Sacc.load_fits(os.path.join(sacc_path, sacc_file_nm)) + likelihood.read(sacc_data) - hmf = ccl.halos.MassFuncTinker08() - min_mass, max_mass = 13.0, 16.0 - min_z, max_z = 0.2, 0.8 - cluster_abundance = ClusterAbundance(min_mass, max_mass, min_z, max_z, hmf) - - # Create and add the kernels you want in your cluster abundance - pivot_mass, pivot_redshift = 14.625862906, 0.6 - mass_observable_kernel = MassRichnessMuSigma(pivot_mass, pivot_redshift) - redshift_proxy_kernel = SpectroscopicRedshift() - # redshift_proxy_kernel = DESY1PhotometricRedshift() - # completeness_kernel = Completeness() - # purity_kernel = Purity() - - cluster_abundance.add_kernel(mass_observable_kernel) - cluster_abundance.add_kernel(redshift_proxy_kernel) - # cluster_abundance.add_kernel(completeness_kernel) - # cluster_abundance.add_kernel(purity_kernel) - - # Put it all together - stats_list = [counts] - lk = ConstGaussian(stats_list) - lk.read(sacc_data) - modeling = ModelingTools(cluster_abundance=cluster_abundance) + sacc_adapter = SaccAdapter( + sacc_data, survey_name, use_cluster_counts, use_mean_log_mass + ) + cluster_abundance = get_cluster_abundance(sacc_adapter.survey_tracer.sky_area) + modeling_tools = ModelingTools(cluster_abundance=cluster_abundance) - return lk, modeling + return likelihood, modeling_tools diff --git a/firecrown/integrator/__init__.py b/firecrown/integrator/__init__.py new file mode 100644 index 000000000..7828e5789 --- /dev/null +++ b/firecrown/integrator/__init__.py @@ -0,0 +1 @@ +from .integrator import NumCosmoIntegrator, ScipyIntegrator, Integrator diff --git a/firecrown/integrator/integrator.py b/firecrown/integrator/integrator.py new file mode 100644 index 000000000..318a6dfb0 --- /dev/null +++ b/firecrown/integrator/integrator.py @@ -0,0 +1,86 @@ +from numcosmo_py import Ncm +from typing import Tuple +from abc import ABC, abstractmethod +from scipy.integrate import nquad +import numpy as np + + +class Integrator(ABC): + @abstractmethod + def integrate(self, integrand, bounds, bounds_map, extra_args): + pass + + +class ScipyIntegrator(Integrator): + def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): + super().__init__() + self._relative_tolerance = relative_tolerance + self._absolute_tolerance = absolute_tolerance + + def integrate(self, integrand, bounds, bounds_map, extra_args): + cc = nquad( + integrand, + ranges=bounds, + args=(extra_args, bounds_map), + opts={ + "epsabs": self._absolute_tolerance, + "epsrel": self._relative_tolerance, + }, + )[0] + + return cc + + +class NumCosmoIntegrator(Integrator): + def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): + super().__init__() + self._relative_tolerance = relative_tolerance + self._absolute_tolerance = absolute_tolerance + + def integrate(self, integrand, bounds, bounds_map, extra_args): + super().__init__() + Ncm.cfg_init() + int_nd = CountsIntegralND( + len(bounds), + integrand, + extra_args, + bounds_map, + ) + int_nd.set_method(Ncm.IntegralNDMethod.P_V) + int_nd.set_reltol(self._relative_tolerance) + int_nd.set_abstol(self._absolute_tolerance) + res = Ncm.Vector.new(1) + err = Ncm.Vector.new(1) + + bl, bu = zip(*bounds) + int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) + return res.get(0) + + +class CountsIntegralND(Ncm.IntegralND): + """Integral subclass used by the ClusterAbundance + to compute the integrals using numcosmo.""" + + def __init__(self, dim, fun, *args): + super().__init__() + self.dim = dim + self.fun = fun + self.args = args + + # pylint: disable-next=arguments-differ + def do_get_dimensions(self) -> Tuple[int, int]: + """Get number of dimensions.""" + return self.dim, 1 + + # pylint: disable-next=arguments-differ + def do_integrand( + self, + x_vec: Ncm.Vector, + dim: int, + npoints: int, + _fdim: int, + fval_vec: Ncm.Vector, + ) -> None: + """Integrand function.""" + x = np.array(x_vec.dup_array()).reshape(npoints, dim) + fval_vec.set_array([self.fun(x_i, *self.args) for x_i in x]) diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index 7b00c7236..34c0afc8f 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -6,10 +6,10 @@ from .statistic import Statistic, DataVector, TheoryVector from .source.source import SourceSystematic from ....modeling_tools import ModelingTools -import pdb import numpy as np -import cProfile -from pstats import SortKey + +# import cProfile +# from pstats import SortKey class ClusterNumberCounts(Statistic): @@ -20,7 +20,7 @@ def __init__( survey_name: str, systematics: Optional[List[SourceSystematic]] = None, ): - self.pr = cProfile.Profile() + # self.pr = cProfile.Profile() super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None @@ -66,8 +66,6 @@ def get_data_vector(self) -> DataVector: def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: # self.pr.enable() - tools.cluster_abundance.sky_area = self.sky_area - theory_vector_list = [] cluster_counts_list = [] diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index 5bded3778..0633c7c8f 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -9,7 +9,7 @@ import pyccl.nl_pt from firecrown.parameters import ParamsMap -from .models.cluster_abundance import ClusterAbundance +from firecrown.models.cluster.abundance import ClusterAbundance class ModelingTools: diff --git a/firecrown/models/cluster/__init__.py b/firecrown/models/cluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/firecrown/models/cluster_abundance.py b/firecrown/models/cluster/abundance.py similarity index 64% rename from firecrown/models/cluster_abundance.py rename to firecrown/models/cluster/abundance.py index 501c96626..3643fe59d 100644 --- a/firecrown/models/cluster_abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,19 +1,14 @@ -from typing import List, Tuple +from typing import List from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl -from scipy.integrate import nquad -from firecrown.models.kernel import Kernel, KernelType, ArgsMapper +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapper import numpy as np -import pdb -from numcosmo_py import Ncm from firecrown.parameters import ParamsMap +from firecrown.integrator import Integrator class ClusterAbundance(object): - _absolute_tolerance = 1e-12 - _relative_tolerance = 1e-4 - @property def sky_area(self) -> float: return self.sky_area_rad * (180.0 / np.pi) ** 2 @@ -33,7 +28,8 @@ def __init__( min_z: float, max_z: float, halo_mass_function: pyccl.halos.MassFunc, - sky_area: float = 0, + sky_area: float, + integrator: Integrator, ): self.kernels: List[Kernel] = [] self.halo_mass_function = halo_mass_function @@ -42,6 +38,7 @@ def __init__( self.min_z = min_z self.max_z = max_z self.sky_area = sky_area + self.integrator = integrator self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel): @@ -83,9 +80,9 @@ def get_abundance_integrand(self): # Use the bounds mapping from the outer scope. def integrand(*int_args): - args_map = int_args[-1] - z = int_args[0][args_map.integral_bounds[KernelType.z.name]] - mass = int_args[0][args_map.integral_bounds[KernelType.mass.name]] + args_map: ArgsMapper = int_args[-1] + z = args_map.get_integral_bounds(int_args, KernelType.z) + mass = args_map.get_integral_bounds(int_args, KernelType.mass) integrand = self.comoving_volume(z) * self.mass_function(mass, z) for kernel in self.kernels: @@ -98,13 +95,16 @@ def integrand(*int_args): return integrand - def get_analytic_kernels(self): + @property + def analytic_kernels(self): return [x for x in self.kernels if x.has_analytic_sln] - def get_dirac_delta_kernels(self): + @property + def dirac_delta_kernels(self): return [x for x in self.kernels if x.is_dirac_delta] - def get_integrable_kernels(self): + @property + def integrable_kernels(self): return [ x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln ] @@ -116,7 +116,7 @@ def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): start_idx = len(args_mapping.integral_bounds.keys()) - for kernel in self.get_dirac_delta_kernels(): + for kernel in self.dirac_delta_kernels: # If any kernel is a dirac delta for z or M, just replace the # True limits with the proxy limits if kernel.kernel_type == KernelType.z_proxy: @@ -124,7 +124,7 @@ def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): elif kernel.kernel_type == KernelType.mass_proxy: bounds_list[0] = mass_proxy_limits - for kernel in self.get_integrable_kernels(): + for kernel in self.integrable_kernels: # If any kernel is not a dirac delta, integrate over the relevant limits if kernel.kernel_type == KernelType.z_proxy: args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx @@ -139,7 +139,7 @@ def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): extra_args = [] start_idx = 0 - for kernel in self.get_analytic_kernels(): + for kernel in self.analytic_kernels: # Lastly, don't integrate any analyticly solved kernels, just solve them. if kernel.kernel_type == KernelType.z_proxy: args_mapping.extra_args[kernel.kernel_type.name] = start_idx @@ -152,76 +152,10 @@ def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): return bounds_list, extra_args, args_mapping - def get_argument_mapper(self): - return - def compute(self, z_proxy_limits, mass_proxy_limits): bounds, extra_args, args_mapping = self.get_integration_bounds( z_proxy_limits, mass_proxy_limits ) - integrand = self.get_abundance_integrand() - # cc = self._scipy_nquad_integrate(integrand, bounds, args_mapping, extra_args) - - cc = self._numcosmo_integrate(integrand, bounds, args_mapping, extra_args) + cc = self.integrator.integrate(integrand, bounds, args_mapping, extra_args) return cc - - def _numcosmo_integrate(self, integrand, bounds, bounds_map, extra_args): - Ncm.cfg_init() - int_nd = CountsIntegralND( - len(bounds), - integrand, - extra_args, - bounds_map, - ) - int_nd.set_method(Ncm.IntegralNDMethod.P_V) - int_nd.set_reltol(self._relative_tolerance) - int_nd.set_abstol(self._absolute_tolerance) - res = Ncm.Vector.new(1) - err = Ncm.Vector.new(1) - - bl, bu = zip(*bounds) - int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) - return res.get(0) - - def _scipy_nquad_integrate(self, integrand, bounds, bounds_map, extra_args): - cc = nquad( - integrand, - ranges=bounds, - args=(extra_args, bounds_map), - opts={ - "epsabs": self._absolute_tolerance, - "epsrel": self._relative_tolerance, - }, - )[0] - - return cc - - -class CountsIntegralND(Ncm.IntegralND): - """Integral subclass used by the ClusterAbundance - to compute the integrals using numcosmo.""" - - def __init__(self, dim, fun, *args): - super().__init__() - self.dim = dim - self.fun = fun - self.args = args - - # pylint: disable-next=arguments-differ - def do_get_dimensions(self) -> Tuple[int, int]: - """Get number of dimensions.""" - return self.dim, 1 - - # pylint: disable-next=arguments-differ - def do_integrand( - self, - x_vec: Ncm.Vector, - dim: int, - npoints: int, - _fdim: int, - fval_vec: Ncm.Vector, - ) -> None: - """Integrand function.""" - x = np.array(x_vec.dup_array()).reshape(npoints, dim) - fval_vec.set_array([self.fun(x_i, *self.args) for x_i in x]) diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py new file mode 100644 index 000000000..b82dd25e4 --- /dev/null +++ b/firecrown/models/cluster/kernel.py @@ -0,0 +1,199 @@ +from abc import ABC +from enum import Enum +from typing import List, Tuple + +import numpy as np +from scipy import special + +from firecrown import parameters +from firecrown.updatable import Updatable + + +class KernelType(Enum): + mass = 1 + z = 2 + mass_proxy = 3 + z_proxy = 4 + completeness = 5 + purity = 6 + + +class ArgsMapper: + def __init__(self): + self.integral_bounds = dict() + self.extra_args = dict() + + self.integral_bounds_idx = 0 + self.extra_args_idx = 1 + + def get_integral_bounds(self, int_args, kernel_type: KernelType): + bounds_values = int_args[self.integral_bounds_idx] + return bounds_values[self.integral_bounds[kernel_type.name]] + + def get_extra_args(self, int_args, kernel_type: KernelType): + extra_values = int_args[self.extra_args_idx] + return extra_values[self.extra_args[kernel_type.name]] + + +class Kernel(Updatable, ABC): + def __init__( + self, + kernel_type: KernelType, + is_dirac_delta=False, + has_analytic_sln=False, + integral_bounds: List[Tuple[float, float]] = None, + ): + super().__init__() + self.integral_bounds = integral_bounds + self.is_dirac_delta = is_dirac_delta + self.kernel_type = kernel_type + self.has_analytic_sln = has_analytic_sln + + def distribution(self, args: List[float], args_map: ArgsMapper): + raise NotImplementedError() + + def analytic_solution(self, args: List[float], args_map: ArgsMapper): + raise NotImplementedError() + + +class Completeness(Kernel): + def __init__(self): + super().__init__(KernelType.completeness) + + def distribution(self, args: List[float], args_map: ArgsMapper): + mass = args_map.get_integral_bounds(args, KernelType.mass) + z = args_map.get_integral_bounds(args, KernelType.z) + + # TODO improve parameter names + a_nc = 1.1321 + b_nc = 0.7751 + a_mc = 13.31 + b_mc = 0.2025 + log_mc = a_mc + b_mc * (1.0 + z) + nc = a_nc + b_nc * (1.0 + z) + completeness = (mass / log_mc) ** nc / ((mass / log_mc) ** nc + 1.0) + return completeness + + +class Purity(Kernel): + def __init__(self): + super().__init__(KernelType.purity) + + def distribution(self, args: List[float], index_lkp: ArgsMapper): + mass_proxy = args[index_lkp.integral_bounds[KernelType.mass_proxy.name]] + z = args[index_lkp.integral_bounds[KernelType.z.name]] + + ln_r = np.log(10**mass_proxy) + a_nc = np.log(10) * 0.8612 + b_nc = np.log(10) * 0.3527 + a_rc = 2.2183 + b_rc = -0.6592 + nc = a_nc + b_nc * (1.0 + z) + ln_rc = a_rc + b_rc * (1.0 + z) + purity = (ln_r / ln_rc) ** nc / ((ln_r / ln_rc) ** nc + 1.0) + return purity + + +class MassRichnessMuSigma(Kernel): + def __init__( + self, + pivot_mass, + pivot_redshift, + integral_bounds: List[Tuple[float, float]] = None, + ): + super().__init__(KernelType.mass_proxy, False, True, integral_bounds) + self.pivot_mass = pivot_mass + self.pivot_redshift = pivot_redshift + self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) + self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) + + # Updatable parameters + self.mu_p0 = parameters.create() + self.mu_p1 = parameters.create() + self.mu_p2 = parameters.create() + self.sigma_p0 = parameters.create() + self.sigma_p1 = parameters.create() + self.sigma_p2 = parameters.create() + + # Verify this gets called last or first + + def observed_value(self, p: Tuple[float, float, float], mass, z): + """Return observed quantity corrected by redshift and mass.""" + + ln_mass = mass * np.log(10) + delta_ln_mass = ln_mass - self.pivot_mass + delta_z = np.log1p(z) - self.log1p_pivot_redshift + + return p[0] + p[1] * delta_ln_mass + p[2] * delta_z + + def analytic_solution(self, args: List[float], args_map: ArgsMapper): + # pdb.set_trace() + + mass = args_map.get_integral_bounds(args, KernelType.mass) + z = args_map.get_integral_bounds(args, KernelType.z) + mass_limits = args_map.get_extra_args(args, self.kernel_type) + + observed_mean_mass = self.observed_value( + (self.mu_p0, self.mu_p1, self.mu_p2), + mass, + z, + ) + observed_mass_sigma = self.observed_value( + (self.sigma_p0, self.sigma_p1, self.sigma_p2), + mass, + z, + ) + + x_min = (observed_mean_mass - mass_limits[0] * np.log(10.0)) / ( + np.sqrt(2.0) * observed_mass_sigma + ) + x_max = (observed_mean_mass - mass_limits[1] * np.log(10.0)) / ( + np.sqrt(2.0) * observed_mass_sigma + ) + + if x_max > 3.0 or x_min < -3.0: + return -(special.erfc(x_min) - special.erfc(x_max)) / 2.0 + return (special.erf(x_min) - special.erf(x_max)) / 2.0 + + # TODO UNDERSTAND THIS + # def spread_point(self, logM: float, z: float, *_) -> float: + # """Return the probability of the point argument.""" + + # lnM_obs = self.logM_obs * np.log(10.0) + + # lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) + # x = lnM_obs - lnM_mu + # chisq = np.dot(x, x) / (2.0 * sigma**2) + # likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) + # return likelihood * np.log(10.0) + + +class TrueMass(Kernel): + def __init__(self): + super().__init__(KernelType.mass_proxy, True) + + def distribution(self, args: List[float], args_map: ArgsMapper): + return 1.0 + + +class SpectroscopicRedshift(Kernel): + def __init__(self): + super().__init__(KernelType.z_proxy, True) + + def distribution(self, args: List[float], args_map: ArgsMapper): + return 1.0 + + +class DESY1PhotometricRedshift(Kernel): + def __init__(self): + super().__init__(KernelType.z_proxy) + self.sigma_0 = 0.05 + + def distribution(self, args: List[float], args_map: ArgsMapper): + z = args_map.get_integral_bounds(args, KernelType.z) + z_proxy = args_map.get_integral_bounds(args, KernelType.z_proxy) + + sigma_z = self.sigma_0 * (1 + z) + prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) + distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) + return prefactor * distribution diff --git a/firecrown/models/kernel.py b/firecrown/models/kernel.py deleted file mode 100644 index b9af85bb1..000000000 --- a/firecrown/models/kernel.py +++ /dev/null @@ -1,83 +0,0 @@ -from typing import List, Tuple, Dict -import numpy as np -from firecrown.updatable import Updatable -from abc import ABC -from enum import Enum -from dataclasses import dataclass, field - - -@dataclass -class ArgsMapper: - integral_bounds: dict = field(default_factory=lambda: {}) - extra_args: dict = field(default_factory=lambda: {}) - - -class KernelType(Enum): - mass = 1 - z = 2 - mass_proxy = 3 - z_proxy = 4 - completeness = 5 - purity = 6 - - -class Kernel(Updatable, ABC): - def __init__( - self, - kernel_type: KernelType, - is_dirac_delta=False, - has_analytic_sln=False, - integral_bounds: List[Tuple[float, float]] = None, - ): - super().__init__() - self.integral_bounds = integral_bounds - self.is_dirac_delta = is_dirac_delta - self.kernel_type = kernel_type - self.has_analytic_sln = has_analytic_sln - - def distribution(self, args: List[float], args_map: ArgsMapper): - raise NotImplementedError() - - def analytic_solution(self, args: List[float], args_map: ArgsMapper): - raise NotImplementedError() - - -class Completeness(Kernel): - def __init__(self): - super().__init__(KernelType.completeness) - - def distribution(self, args: List[float], args_map: ArgsMapper): - mass = args[args_map.integral_bounds[KernelType.mass.name]] - z = args[args_map.integral_bounds[KernelType.z.name]] - # TODO improve parameter names - a_nc = 1.1321 - b_nc = 0.7751 - a_mc = 13.31 - b_mc = 0.2025 - log_mc = a_mc + b_mc * (1.0 + z) - nc = a_nc + b_nc * (1.0 + z) - completeness = (mass / log_mc) ** nc / ((mass / log_mc) ** nc + 1.0) - return completeness - - -class Purity(Kernel): - def __init__(self): - super().__init__(KernelType.purity) - - def distribution(self, args: List[float], index_lkp: ArgsMapper): - mass_proxy = args[index_lkp.integral_bounds[KernelType.mass_proxy.name]] - z = args[index_lkp.integral_bounds[KernelType.z.name]] - - ln_r = np.log(10**mass_proxy) - a_nc = np.log(10) * 0.8612 - b_nc = np.log(10) * 0.3527 - a_rc = 2.2183 - b_rc = -0.6592 - nc = a_nc + b_nc * (1.0 + z) - ln_rc = a_rc + b_rc * (1.0 + z) - purity = (ln_r / ln_rc) ** nc / ((ln_r / ln_rc) ** nc + 1.0) - return purity - - -class Miscentering(Kernel): - pass diff --git a/firecrown/models/mass_observable.py b/firecrown/models/mass_observable.py deleted file mode 100644 index 6fcbc6165..000000000 --- a/firecrown/models/mass_observable.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Cluster Mass Richness proxy module - -Define the Cluster Mass Richness proxy module and its arguments. -""" -from typing import Tuple, List, Dict - -import numpy as np -from scipy import special - -from .. import parameters -from .kernel import Kernel, KernelType, ArgsMapper -import pdb - - -class MassRichnessMuSigma(Kernel): - def __init__( - self, - pivot_mass, - pivot_redshift, - integral_bounds: List[Tuple[float, float]] = None, - ): - super().__init__(KernelType.mass_proxy, False, True, integral_bounds) - self.pivot_mass = pivot_mass - self.pivot_redshift = pivot_redshift - self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) - self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) - - # Updatable parameters - self.mu_p0 = parameters.create() - self.mu_p1 = parameters.create() - self.mu_p2 = parameters.create() - self.sigma_p0 = parameters.create() - self.sigma_p1 = parameters.create() - self.sigma_p2 = parameters.create() - - # Verify this gets called last or first - - def observed_value(self, p: Tuple[float, float, float], mass, z): - """Return observed quantity corrected by redshift and mass.""" - - ln_mass = mass * np.log(10) - delta_ln_mass = ln_mass - self.pivot_mass - delta_z = np.log1p(z) - self.log1p_pivot_redshift - - return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - - def analytic_solution(self, args: List[float], index_lkp: ArgsMapper): - # pdb.set_trace() - bounds = args[0] - extra_args = args[1] - mass = bounds[index_lkp.integral_bounds[KernelType.mass.name]] - z = bounds[index_lkp.integral_bounds[KernelType.z.name]] - mass_limits = extra_args[index_lkp.extra_args[self.kernel_type.name]] - - observed_mean_mass = self.observed_value( - (self.mu_p0, self.mu_p1, self.mu_p2), - mass, - z, - ) - observed_mass_sigma = self.observed_value( - (self.sigma_p0, self.sigma_p1, self.sigma_p2), - mass, - z, - ) - - x_min = (observed_mean_mass - mass_limits[0] * np.log(10.0)) / ( - np.sqrt(2.0) * observed_mass_sigma - ) - x_max = (observed_mean_mass - mass_limits[1] * np.log(10.0)) / ( - np.sqrt(2.0) * observed_mass_sigma - ) - - if x_max > 3.0 or x_min < -3.0: - return -(special.erfc(x_min) - special.erfc(x_max)) / 2.0 - return (special.erf(x_min) - special.erf(x_max)) / 2.0 - - # TODO UNDERSTAND THIS - # def spread_point(self, logM: float, z: float, *_) -> float: - # """Return the probability of the point argument.""" - - # lnM_obs = self.logM_obs * np.log(10.0) - - # lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) - # x = lnM_obs - lnM_mu - # chisq = np.dot(x, x) / (2.0 * sigma**2) - # likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) - # return likelihood * np.log(10.0) - - -class TrueMass(Kernel): - def __init__(self): - super().__init__(KernelType.mass_proxy, True) - - def distribution(self, args: List[float], index_lkp: ArgsMapper): - return 1.0 diff --git a/firecrown/models/redshift.py b/firecrown/models/redshift.py deleted file mode 100644 index b88e80992..000000000 --- a/firecrown/models/redshift.py +++ /dev/null @@ -1,26 +0,0 @@ -import numpy as np -from firecrown.models.kernel import Kernel, KernelType, ArgsMapper -from typing import List, Tuple, Dict - - -class SpectroscopicRedshift(Kernel): - def __init__(self): - super().__init__(KernelType.z_proxy, True) - - def distribution(self, args: List[float], index_lkp: ArgsMapper): - return 1.0 - - -class DESY1PhotometricRedshift(Kernel): - def __init__(self): - super().__init__(KernelType.z_proxy) - self.sigma_0 = 0.05 - - def distribution(self, args: List[float], index_lkp: ArgsMapper): - z_proxy = args[index_lkp[KernelType.z_proxy.name]] - z = args[index_lkp[KernelType.z.name]] - - sigma_z = self.sigma_0 * (1 + z) - prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) - distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) - return prefactor * distribution From a5b7ed17cd02ac2fb580fd716ebb9a49d4a4aba5 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 13 Oct 2023 11:48:30 -0700 Subject: [PATCH 12/80] Tried to tidy up the complicated integral bounds method. --- firecrown/integrator/integrator.py | 3 + firecrown/models/cluster/abundance.py | 98 +++++++++++++-------------- firecrown/models/cluster/kernel.py | 18 ++--- 3 files changed, 61 insertions(+), 58 deletions(-) diff --git a/firecrown/integrator/integrator.py b/firecrown/integrator/integrator.py index 318a6dfb0..91f2e2a1b 100644 --- a/firecrown/integrator/integrator.py +++ b/firecrown/integrator/integrator.py @@ -18,6 +18,9 @@ def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): self._absolute_tolerance = absolute_tolerance def integrate(self, integrand, bounds, bounds_map, extra_args): + # TODO: Scipy passes the bounds unwrapped, while NumCosmo passes the bounds + # Wrapped. This means we need to adjust the called code to handle this. + # This isn't done yet. cc = nquad( integrand, ranges=bounds, diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 3643fe59d..005706818 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -2,7 +2,7 @@ from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapper +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping import numpy as np from firecrown.parameters import ParamsMap from firecrown.integrator import Integrator @@ -21,6 +21,20 @@ def sky_area(self, sky_area: float) -> None: def cosmo(self) -> Cosmology: return self._cosmo + @property + def analytic_kernels(self): + return [x for x in self.kernels if x.has_analytic_sln] + + @property + def dirac_delta_kernels(self): + return [x for x in self.kernels if x.is_dirac_delta] + + @property + def integrable_kernels(self): + return [ + x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln + ] + def __init__( self, min_mass: float, @@ -77,10 +91,8 @@ def mass_function(self, mass: float, z: float) -> float: return hmf def get_abundance_integrand(self): - # Use the bounds mapping from the outer scope. - def integrand(*int_args): - args_map: ArgsMapper = int_args[-1] + args_map: ArgsMapping = int_args[-1] z = args_map.get_integral_bounds(int_args, KernelType.z) mass = args_map.get_integral_bounds(int_args, KernelType.mass) @@ -95,62 +107,50 @@ def integrand(*int_args): return integrand - @property - def analytic_kernels(self): - return [x for x in self.kernels if x.has_analytic_sln] - - @property - def dirac_delta_kernels(self): - return [x for x in self.kernels if x.is_dirac_delta] - - @property - def integrable_kernels(self): - return [ - x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln - ] - def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): - args_mapping = ArgsMapper() + args_mapping = ArgsMapping() args_mapping.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - bounds_list = [(self.min_mass, self.max_mass), (self.min_z, self.max_z)] - - start_idx = len(args_mapping.integral_bounds.keys()) + integral_bounds = [(self.min_mass, self.max_mass), (self.min_z, self.max_z)] + extra_args = [] for kernel in self.dirac_delta_kernels: # If any kernel is a dirac delta for z or M, just replace the - # True limits with the proxy limits + # true limits with the proxy limits if kernel.kernel_type == KernelType.z_proxy: - bounds_list[1] = z_proxy_limits + integral_bounds[1] = z_proxy_limits elif kernel.kernel_type == KernelType.mass_proxy: - bounds_list[0] = mass_proxy_limits + integral_bounds[0] = mass_proxy_limits + mapping_idx = len(args_mapping.integral_bounds.keys()) for kernel in self.integrable_kernels: # If any kernel is not a dirac delta, integrate over the relevant limits - if kernel.kernel_type == KernelType.z_proxy: - args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx - bounds_list.append(z_proxy_limits) - elif kernel.kernel_type == KernelType.mass_proxy: - args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx - bounds_list.append(mass_proxy_limits) - else: - args_mapping.integral_bounds[kernel.kernel_type.name] = start_idx - bounds_list.append(kernel.integral_bounds) - start_idx += 1 - - extra_args = [] - start_idx = 0 + args_mapping.integral_bounds[kernel.kernel_type.name] = mapping_idx + mapping_idx += 1 + + match kernel.kernel_type: + case KernelType.z_proxy: + integral_bounds.append(z_proxy_limits) + case KernelType.mass_proxy: + integral_bounds.append(mass_proxy_limits) + case _: + integral_bounds.append(kernel.integral_bounds) + + mapping_idx = 0 for kernel in self.analytic_kernels: - # Lastly, don't integrate any analyticly solved kernels, just solve them. - if kernel.kernel_type == KernelType.z_proxy: - args_mapping.extra_args[kernel.kernel_type.name] = start_idx - extra_args.append(z_proxy_limits) - elif kernel.kernel_type == KernelType.mass_proxy: - args_mapping.extra_args[kernel.kernel_type.name] = start_idx - extra_args.append(mass_proxy_limits) - - start_idx += 1 - - return bounds_list, extra_args, args_mapping + # Lastly, don't integrate any kernels with an analytic solution + # This means we pass in their limits as extra arguments + args_mapping.extra_args[kernel.kernel_type.name] = mapping_idx + mapping_idx += 1 + + match kernel.kernel_type: + case KernelType.z_proxy: + extra_args.append(z_proxy_limits) + case KernelType.mass_proxy: + extra_args.append(mass_proxy_limits) + case _: + extra_args.append(kernel.integral_bounds) + + return integral_bounds, extra_args, args_mapping def compute(self, z_proxy_limits, mass_proxy_limits): bounds, extra_args, args_mapping = self.get_integration_bounds( diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index b82dd25e4..4fa3eceb1 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -18,7 +18,7 @@ class KernelType(Enum): purity = 6 -class ArgsMapper: +class ArgsMapping: def __init__(self): self.integral_bounds = dict() self.extra_args = dict() @@ -49,10 +49,10 @@ def __init__( self.kernel_type = kernel_type self.has_analytic_sln = has_analytic_sln - def distribution(self, args: List[float], args_map: ArgsMapper): + def distribution(self, args: List[float], args_map: ArgsMapping): raise NotImplementedError() - def analytic_solution(self, args: List[float], args_map: ArgsMapper): + def analytic_solution(self, args: List[float], args_map: ArgsMapping): raise NotImplementedError() @@ -60,7 +60,7 @@ class Completeness(Kernel): def __init__(self): super().__init__(KernelType.completeness) - def distribution(self, args: List[float], args_map: ArgsMapper): + def distribution(self, args: List[float], args_map: ArgsMapping): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) @@ -79,7 +79,7 @@ class Purity(Kernel): def __init__(self): super().__init__(KernelType.purity) - def distribution(self, args: List[float], index_lkp: ArgsMapper): + def distribution(self, args: List[float], index_lkp: ArgsMapping): mass_proxy = args[index_lkp.integral_bounds[KernelType.mass_proxy.name]] z = args[index_lkp.integral_bounds[KernelType.z.name]] @@ -126,7 +126,7 @@ def observed_value(self, p: Tuple[float, float, float], mass, z): return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - def analytic_solution(self, args: List[float], args_map: ArgsMapper): + def analytic_solution(self, args: List[float], args_map: ArgsMapping): # pdb.set_trace() mass = args_map.get_integral_bounds(args, KernelType.mass) @@ -172,7 +172,7 @@ class TrueMass(Kernel): def __init__(self): super().__init__(KernelType.mass_proxy, True) - def distribution(self, args: List[float], args_map: ArgsMapper): + def distribution(self, args: List[float], args_map: ArgsMapping): return 1.0 @@ -180,7 +180,7 @@ class SpectroscopicRedshift(Kernel): def __init__(self): super().__init__(KernelType.z_proxy, True) - def distribution(self, args: List[float], args_map: ArgsMapper): + def distribution(self, args: List[float], args_map: ArgsMapping): return 1.0 @@ -189,7 +189,7 @@ def __init__(self): super().__init__(KernelType.z_proxy) self.sigma_0 = 0.05 - def distribution(self, args: List[float], args_map: ArgsMapper): + def distribution(self, args: List[float], args_map: ArgsMapping): z = args_map.get_integral_bounds(args, KernelType.z) z_proxy = args_map.get_integral_bounds(args, KernelType.z_proxy) From 442d547c999eae38ed44f446cdabe950f32b8a9e Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 13 Oct 2023 14:29:28 -0700 Subject: [PATCH 13/80] Small changes to support the mean mass calculation. Noticed a bug? in the previous code with limits for integration. --- .../cluster_redshift_richness.py | 4 +- .../statistic/cluster_number_counts.py | 57 ++++++++++--------- firecrown/models/cluster/abundance.py | 17 +++++- .../abundance_data.py} | 2 +- 4 files changed, 48 insertions(+), 32 deletions(-) rename firecrown/models/{sacc_adapter.py => cluster/abundance_data.py} (99%) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 471741c84..8445d1024 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -10,7 +10,7 @@ from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( ClusterNumberCounts, ) -from firecrown.models.sacc_adapter import SaccAdapter +from firecrown.models.cluster.abundance_data import AbundanceData from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import ( @@ -70,7 +70,7 @@ def build_likelihood(build_parameters): sacc_data = sacc.Sacc.load_fits(os.path.join(sacc_path, sacc_file_nm)) likelihood.read(sacc_data) - sacc_adapter = SaccAdapter( + sacc_adapter = AbundanceData( sacc_data, survey_name, use_cluster_counts, use_mean_log_mass ) cluster_abundance = get_cluster_abundance(sacc_adapter.survey_tracer.sky_area) diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index 34c0afc8f..3db741d69 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -2,13 +2,14 @@ from typing import List, Optional import sacc -from firecrown.models.sacc_adapter import SaccAdapter +from firecrown.models.cluster.abundance_data import AbundanceData from .statistic import Statistic, DataVector, TheoryVector from .source.source import SourceSystematic from ....modeling_tools import ModelingTools import numpy as np -# import cProfile +import cProfile + # from pstats import SortKey @@ -20,7 +21,7 @@ def __init__( survey_name: str, systematics: Optional[List[SourceSystematic]] = None, ): - # self.pr = cProfile.Profile() + self.pr = cProfile.Profile() super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None @@ -34,15 +35,14 @@ def read(self, sacc_data: sacc.Sacc): data_vector = [] sacc_indices = [] - bin_limits = [] + sacc_types = sacc.data_types.standard_types - sacc_adapter = SaccAdapter( + sacc_adapter = AbundanceData( sacc_data, self.survey_name, self.use_cluster_counts, self.use_mean_log_mass ) if self.use_cluster_counts: data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) - bin_limits += sacc_adapter.get_bin_limits(sacc_types.cluster_counts) data_vector += data sacc_indices += indices @@ -50,13 +50,15 @@ def read(self, sacc_data: sacc.Sacc): data, indices = sacc_adapter.get_data_and_indices( sacc_types.cluster_mean_log_mass ) - bin_limits += sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) data_vector += data sacc_indices += indices self.sky_area = sacc_adapter.survey_tracer.sky_area - self.bin_limits = bin_limits + # Note - this is the same for both cl mass and cl counts... Why do we need to + # specify a data type? + self.bin_limits = sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) self.data_vector = DataVector.from_list(data_vector) + print(len(data_vector)) self.sacc_indices = np.array(sacc_indices) super().read(sacc_data) @@ -67,28 +69,31 @@ def get_data_vector(self) -> DataVector: def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: # self.pr.enable() theory_vector_list = [] - cluster_counts_list = [] + cluster_counts = [] + cluster_masses = [] - if self.use_cluster_counts: + if self.use_cluster_counts or self.use_mean_log_mass: for z_proxy_limits, mass_proxy_limits in self.bin_limits: - cluster_counts = tools.cluster_abundance.compute( + counts = tools.cluster_abundance.compute_counts( z_proxy_limits, mass_proxy_limits ) - cluster_counts_list.append(cluster_counts) - theory_vector_list += cluster_counts_list - - # if self.use_mean_log_mass: - # mean_log_mass_list = [ - # self.cluster_abundance.compute_unormalized_mean_logM( - # ccl_cosmo, logM_tracer_arg, z_tracer_arg - # ) - # / counts - # for (z_tracer_arg, logM_tracer_arg), counts in zip( - # self.tracer_args, cluster_counts_list - # ) - # ] - # theory_vector_list += mean_log_mass_list + cluster_counts.append(counts) + theory_vector_list += cluster_counts + + if self.use_mean_log_mass: + for (z_proxy_limits, mass_proxy_limits), counts in zip( + self.bin_limits, cluster_counts + ): + cluster_mass = ( + tools.cluster_abundance.compute_mass( + z_proxy_limits, mass_proxy_limits + ) + / counts + ) + cluster_masses.append(cluster_mass) + theory_vector_list += cluster_masses + # self.pr.disable() # self.pr.dump_stats("profile.prof") - + print(len(theory_vector_list)) return TheoryVector.from_list(theory_vector_list) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 005706818..47c730fdf 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -6,6 +6,7 @@ import numpy as np from firecrown.parameters import ParamsMap from firecrown.integrator import Integrator +import pdb class ClusterAbundance(object): @@ -90,13 +91,15 @@ def mass_function(self, mass: float, z: float) -> float: hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) return hmf - def get_abundance_integrand(self): + def get_integrand(self, include_mass=False): def integrand(*int_args): args_map: ArgsMapping = int_args[-1] z = args_map.get_integral_bounds(int_args, KernelType.z) mass = args_map.get_integral_bounds(int_args, KernelType.mass) integrand = self.comoving_volume(z) * self.mass_function(mass, z) + if include_mass: + integrand *= mass for kernel in self.kernels: integrand *= ( kernel.analytic_solution(int_args, args_map) @@ -152,10 +155,18 @@ def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): return integral_bounds, extra_args, args_mapping - def compute(self, z_proxy_limits, mass_proxy_limits): + def compute_counts(self, z_proxy_limits, mass_proxy_limits): bounds, extra_args, args_mapping = self.get_integration_bounds( z_proxy_limits, mass_proxy_limits ) - integrand = self.get_abundance_integrand() + integrand = self.get_integrand() + cc = self.integrator.integrate(integrand, bounds, args_mapping, extra_args) + return cc + + def compute_mass(self, z_proxy_limits, mass_proxy_limits): + bounds, extra_args, args_mapping = self.get_integration_bounds( + z_proxy_limits, mass_proxy_limits + ) + integrand = self.get_integrand(include_mass=True) cc = self.integrator.integrate(integrand, bounds, args_mapping, extra_args) return cc diff --git a/firecrown/models/sacc_adapter.py b/firecrown/models/cluster/abundance_data.py similarity index 99% rename from firecrown/models/sacc_adapter.py rename to firecrown/models/cluster/abundance_data.py index f74e99d3a..548adb2b8 100644 --- a/firecrown/models/sacc_adapter.py +++ b/firecrown/models/cluster/abundance_data.py @@ -3,7 +3,7 @@ from sacc.tracers import SurveyTracer -class SaccAdapter: +class AbundanceData: # Hard coded in SACC, how do we want to handle this? _survey_index = 0 _redshift_index = 1 From eb44d3f413d262d7ce216d6d3eb0de43afae793e Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 16 Oct 2023 15:50:05 -0700 Subject: [PATCH 14/80] Updated numcosmo integration to pass in vectors Updated rest of the abundance code to do vector operations Implemented a cache for HMF for now since it doesn't accept a vector of scale factors. --- firecrown/integrator/integrator.py | 2 +- .../statistic/cluster_number_counts.py | 3 -- firecrown/models/cluster/abundance.py | 34 +++++++++++++++---- firecrown/models/cluster/kernel.py | 29 +++++++--------- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/firecrown/integrator/integrator.py b/firecrown/integrator/integrator.py index 91f2e2a1b..75f6f77fe 100644 --- a/firecrown/integrator/integrator.py +++ b/firecrown/integrator/integrator.py @@ -86,4 +86,4 @@ def do_integrand( ) -> None: """Integrand function.""" x = np.array(x_vec.dup_array()).reshape(npoints, dim) - fval_vec.set_array([self.fun(x_i, *self.args) for x_i in x]) + fval_vec.set_array(self.fun(x, *self.args)) diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py index 3db741d69..c4e58b948 100644 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py @@ -67,7 +67,6 @@ def get_data_vector(self) -> DataVector: return self.data_vector def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: - # self.pr.enable() theory_vector_list = [] cluster_counts = [] cluster_masses = [] @@ -93,7 +92,5 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_masses.append(cluster_mass) theory_vector_list += cluster_masses - # self.pr.disable() - # self.pr.dump_stats("profile.prof") print(len(theory_vector_list)) return TheoryVector.from_list(theory_vector_list) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 47c730fdf..2beb6491a 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -6,7 +6,16 @@ import numpy as np from firecrown.parameters import ParamsMap from firecrown.integrator import Integrator -import pdb +import numpy.typing as npt + +# TODO: 1. CCL recomputing and not caching - mass function +# 2. Integrator arguments issues +# * mass_function will ultimately be an NxN grid of values +# This is probably being recomputed for every z, m bin +# * Power spectrum takes the heavy lifting, this will be computed for a +# single +# cosmology. +# * Compute the grid of values for the mass function, then integrate class ClusterAbundance(object): @@ -54,6 +63,7 @@ def __init__( self.max_z = max_z self.sky_area = sky_area self.integrator = integrator + self._hmf_cache = {} self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel): @@ -61,6 +71,7 @@ def add_kernel(self, kernel: Kernel): def update_ingredients(self, cosmo: Cosmology, params: ParamsMap): self._cosmo = cosmo + self._hmf_cache = {} for kernel in self.kernels: kernel.update(params) @@ -86,21 +97,31 @@ def comoving_volume(self, z) -> float: ) return dV * self.sky_area_rad - def mass_function(self, mass: float, z: float) -> float: + def mass_function( + self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] + ) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) - hmf = self.halo_mass_function(self.cosmo, 10**mass, scale_factor) - return hmf + + return_vals = [] + for m, a in zip(mass, scale_factor): + val = self._hmf_cache.get((m, a)) + if val is None: + val = self.halo_mass_function(self.cosmo, 10**m, a) + self._hmf_cache[(m, a)] = val + return_vals.append(val) + + return return_vals def get_integrand(self, include_mass=False): def integrand(*int_args): args_map: ArgsMapping = int_args[-1] z = args_map.get_integral_bounds(int_args, KernelType.z) mass = args_map.get_integral_bounds(int_args, KernelType.mass) - integrand = self.comoving_volume(z) * self.mass_function(mass, z) if include_mass: integrand *= mass for kernel in self.kernels: + # Think of overhead here, if its worth it integrand *= ( kernel.analytic_solution(int_args, args_map) if kernel.has_analytic_sln @@ -141,7 +162,7 @@ def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): mapping_idx = 0 for kernel in self.analytic_kernels: # Lastly, don't integrate any kernels with an analytic solution - # This means we pass in their limits as extra arguments + # This means we pass in their limits as extra arguments to the integrator args_mapping.extra_args[kernel.kernel_type.name] = mapping_idx mapping_idx += 1 @@ -163,6 +184,7 @@ def compute_counts(self, z_proxy_limits, mass_proxy_limits): cc = self.integrator.integrate(integrand, bounds, args_mapping, extra_args) return cc + # compute_average(AverageType.shear, etc) def compute_mass(self, z_proxy_limits, mass_proxy_limits): bounds, extra_args, args_mapping = self.get_integration_bounds( z_proxy_limits, mass_proxy_limits diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 4fa3eceb1..b49c1cb88 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -7,6 +7,7 @@ from firecrown import parameters from firecrown.updatable import Updatable +import pdb class KernelType(Enum): @@ -28,7 +29,7 @@ def __init__(self): def get_integral_bounds(self, int_args, kernel_type: KernelType): bounds_values = int_args[self.integral_bounds_idx] - return bounds_values[self.integral_bounds[kernel_type.name]] + return bounds_values[:, self.integral_bounds[kernel_type.name]] def get_extra_args(self, int_args, kernel_type: KernelType): extra_values = int_args[self.extra_args_idx] @@ -127,12 +128,9 @@ def observed_value(self, p: Tuple[float, float, float], mass, z): return p[0] + p[1] * delta_ln_mass + p[2] * delta_z def analytic_solution(self, args: List[float], args_map: ArgsMapping): - # pdb.set_trace() - mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) mass_limits = args_map.get_extra_args(args, self.kernel_type) - observed_mean_mass = self.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), mass, @@ -151,21 +149,18 @@ def analytic_solution(self, args: List[float], args_map: ArgsMapping): np.sqrt(2.0) * observed_mass_sigma ) - if x_max > 3.0 or x_min < -3.0: - return -(special.erfc(x_min) - special.erfc(x_max)) / 2.0 - return (special.erf(x_min) - special.erf(x_max)) / 2.0 + return_vals = np.empty_like(x_min) + mask1 = (x_max > 3.0) | (x_min < -3.0) + mask2 = ~mask1 - # TODO UNDERSTAND THIS - # def spread_point(self, logM: float, z: float, *_) -> float: - # """Return the probability of the point argument.""" - - # lnM_obs = self.logM_obs * np.log(10.0) + return_vals[mask1] = ( + -(special.erfc(x_min[mask1]) - special.erfc(x_max[mask1])) / 2.0 + ) + return_vals[mask2] = ( + special.erf(x_min[mask2]) - special.erf(x_max[mask2]) + ) / 2.0 - # lnM_mu, sigma = self.richness.cluster_mass_lnM_obs_mu_sigma(logM, z) - # x = lnM_obs - lnM_mu - # chisq = np.dot(x, x) / (2.0 * sigma**2) - # likelihood = np.exp(-chisq) / (np.sqrt(2.0 * np.pi * sigma**2)) - # return likelihood * np.log(10.0) + return return_vals class TrueMass(Kernel): From 23d2707b7081a53590be2ff8b2aad31589b73071 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 17 Oct 2023 08:58:07 -0700 Subject: [PATCH 15/80] Refactored the integration limits and integrator out of the cluster abundance. --- .../cluster_redshift_richness.py | 14 +- .../statistic/binned_cluster_number_counts.py | 156 ++++++++++++++++++ .../statistic/cluster_number_counts.py | 96 ----------- firecrown/models/cluster/abundance.py | 82 ++------- firecrown/models/cluster/kernel.py | 5 +- tests/test_cluster_abundance.py | 4 +- 6 files changed, 178 insertions(+), 179 deletions(-) create mode 100644 firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py delete mode 100644 firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 8445d1024..a794796f9 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -7,8 +7,8 @@ from firecrown.integrator.integrator import NumCosmoIntegrator from firecrown.likelihood.gauss_family.gaussian import ConstGaussian -from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( - ClusterNumberCounts, +from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( + BinnedClusterNumberCounts, ) from firecrown.models.cluster.abundance_data import AbundanceData from firecrown.modeling_tools import ModelingTools @@ -23,12 +23,11 @@ def get_cluster_abundance(sky_area): - integrator = NumCosmoIntegrator() hmf = ccl.halos.MassFuncTinker08() min_mass, max_mass = 13.0, 16.0 min_z, max_z = 0.2, 0.8 cluster_abundance = ClusterAbundance( - min_mass, max_mass, min_z, max_z, hmf, sky_area, integrator + min_mass, max_mass, min_z, max_z, hmf, sky_area ) # Create and add the kernels you want in your cluster abundance @@ -53,13 +52,18 @@ def build_likelihood(build_parameters): """ Here we instantiate the number density (or mass function) object. """ + integrator = NumCosmoIntegrator() # Pull params for the likelihood from build_parameters use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) survey_name = "numcosmo_simulated_redshift_richness" likelihood = ConstGaussian( - [ClusterNumberCounts(use_cluster_counts, use_mean_log_mass, survey_name)] + [ + BinnedClusterNumberCounts( + use_cluster_counts, use_mean_log_mass, survey_name, integrator + ) + ] ) # Read in sacc data diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py new file mode 100644 index 000000000..0c404800b --- /dev/null +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -0,0 +1,156 @@ +from __future__ import annotations +from typing import List, Optional +import sacc + +from firecrown.integrator import Integrator +from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.abundance_data import AbundanceData +from firecrown.models.cluster.kernel import ArgsMapping, KernelType +from .statistic import Statistic, DataVector, TheoryVector +from .source.source import SourceSystematic +from ....modeling_tools import ModelingTools +import numpy as np + + +class BinnedClusterNumberCounts(Statistic): + def __init__( + self, + cluster_counts: bool, + mean_log_mass: bool, + survey_name: str, + integrator: Integrator, + systematics: Optional[List[SourceSystematic]] = None, + ): + super().__init__() + self.systematics = systematics or [] + self.theory_vector: Optional[TheoryVector] = None + self.use_cluster_counts = cluster_counts + self.use_mean_log_mass = mean_log_mass + self.survey_name = survey_name + self.integrator = integrator + self.data_vector = DataVector.from_list([]) + + def read(self, sacc_data: sacc.Sacc): + # Build the data vector and indices needed for the likelihood + + data_vector = [] + sacc_indices = [] + + sacc_types = sacc.data_types.standard_types + sacc_adapter = AbundanceData( + sacc_data, self.survey_name, self.use_cluster_counts, self.use_mean_log_mass + ) + + if self.use_cluster_counts: + data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) + data_vector += data + sacc_indices += indices + + if self.use_mean_log_mass: + data, indices = sacc_adapter.get_data_and_indices( + sacc_types.cluster_mean_log_mass + ) + data_vector += data + sacc_indices += indices + + self.sky_area = sacc_adapter.survey_tracer.sky_area + # Note - this is the same for both cl mass and cl counts... Why do we need to + # specify a data type? + self.bin_limits = sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) + self.data_vector = DataVector.from_list(data_vector) + print(len(data_vector)) + self.sacc_indices = np.array(sacc_indices) + super().read(sacc_data) + + def get_data_vector(self) -> DataVector: + assert self.data_vector is not None + return self.data_vector + + def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: + theory_vector_list = [] + cluster_counts = [] + cluster_masses = [] + + if self.use_cluster_counts or self.use_mean_log_mass: + for z_proxy_limits, mass_proxy_limits in self.bin_limits: + bounds, extra_args, args_mapping = self.get_integration_bounds( + tools.cluster_abundance, z_proxy_limits, mass_proxy_limits + ) + + integrand = tools.cluster_abundance.get_integrand() + counts = self.integrator.integrate( + integrand, bounds, args_mapping, extra_args + ) + + cluster_counts.append(counts) + + theory_vector_list += cluster_counts + + if self.use_mean_log_mass: + for (z_proxy_limits, mass_proxy_limits), counts in zip( + self.bin_limits, cluster_counts + ): + bounds, extra_args, args_mapping = self.get_integration_bounds( + z_proxy_limits, mass_proxy_limits + ) + + integrand = tools.cluster_abundance.get_integrand() + unnormalized_mass = self.integrator.integrate( + integrand, bounds, args_mapping, extra_args + ) + + cluster_mass = unnormalized_mass / counts + cluster_masses.append(cluster_mass) + + theory_vector_list += cluster_masses + + return TheoryVector.from_list(theory_vector_list) + + def get_integration_bounds( + self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits + ): + args_mapping = ArgsMapping() + args_mapping.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + integral_bounds = [ + (cl_abundance.min_mass, cl_abundance.max_mass), + (cl_abundance.min_z, cl_abundance.max_z), + ] + + # If any kernel is a dirac delta for z or M, just replace the + # true limits with the proxy limits + for kernel in cl_abundance.dirac_delta_kernels: + if kernel.kernel_type == KernelType.z_proxy: + integral_bounds[1] = z_proxy_limits + elif kernel.kernel_type == KernelType.mass_proxy: + integral_bounds[0] = mass_proxy_limits + + # If any kernel is not a dirac delta, integrate over the relevant limits + mapping_idx = len(args_mapping.integral_bounds.keys()) + for kernel in cl_abundance.integrable_kernels: + args_mapping.integral_bounds[kernel.kernel_type.name] = mapping_idx + mapping_idx += 1 + + if kernel.kernel_type == KernelType.z_proxy: + integral_bounds.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + integral_bounds.append(mass_proxy_limits) + + if kernel.integral_bounds is not None: + integral_bounds.append(kernel.integral_bounds) + + # Lastly, don't integrate any kernels with an analytic solution + # This means we pass in their limits as extra arguments to the integrator + extra_args = [] + for i, kernel in enumerate(cl_abundance.analytic_kernels): + args_mapping.extra_args[kernel.kernel_type.name] = i + + if kernel.kernel_type == KernelType.z_proxy: + extra_args.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + extra_args.append(mass_proxy_limits) + + if kernel.integral_bounds is not None: + extra_args.append(kernel.integral_bounds) + + return integral_bounds, extra_args, args_mapping diff --git a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py deleted file mode 100644 index c4e58b948..000000000 --- a/firecrown/likelihood/gauss_family/statistic/cluster_number_counts.py +++ /dev/null @@ -1,96 +0,0 @@ -from __future__ import annotations -from typing import List, Optional -import sacc - -from firecrown.models.cluster.abundance_data import AbundanceData -from .statistic import Statistic, DataVector, TheoryVector -from .source.source import SourceSystematic -from ....modeling_tools import ModelingTools -import numpy as np - -import cProfile - -# from pstats import SortKey - - -class ClusterNumberCounts(Statistic): - def __init__( - self, - cluster_counts: bool, - mean_log_mass: bool, - survey_name: str, - systematics: Optional[List[SourceSystematic]] = None, - ): - self.pr = cProfile.Profile() - super().__init__() - self.systematics = systematics or [] - self.theory_vector: Optional[TheoryVector] = None - self.use_cluster_counts = cluster_counts - self.use_mean_log_mass = mean_log_mass - self.survey_name = survey_name - self.data_vector = DataVector.from_list([]) - - def read(self, sacc_data: sacc.Sacc): - # Build the data vector and indices needed for the likelihood - - data_vector = [] - sacc_indices = [] - - sacc_types = sacc.data_types.standard_types - sacc_adapter = AbundanceData( - sacc_data, self.survey_name, self.use_cluster_counts, self.use_mean_log_mass - ) - - if self.use_cluster_counts: - data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) - data_vector += data - sacc_indices += indices - - if self.use_mean_log_mass: - data, indices = sacc_adapter.get_data_and_indices( - sacc_types.cluster_mean_log_mass - ) - data_vector += data - sacc_indices += indices - - self.sky_area = sacc_adapter.survey_tracer.sky_area - # Note - this is the same for both cl mass and cl counts... Why do we need to - # specify a data type? - self.bin_limits = sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) - self.data_vector = DataVector.from_list(data_vector) - print(len(data_vector)) - self.sacc_indices = np.array(sacc_indices) - super().read(sacc_data) - - def get_data_vector(self) -> DataVector: - assert self.data_vector is not None - return self.data_vector - - def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: - theory_vector_list = [] - cluster_counts = [] - cluster_masses = [] - - if self.use_cluster_counts or self.use_mean_log_mass: - for z_proxy_limits, mass_proxy_limits in self.bin_limits: - counts = tools.cluster_abundance.compute_counts( - z_proxy_limits, mass_proxy_limits - ) - cluster_counts.append(counts) - theory_vector_list += cluster_counts - - if self.use_mean_log_mass: - for (z_proxy_limits, mass_proxy_limits), counts in zip( - self.bin_limits, cluster_counts - ): - cluster_mass = ( - tools.cluster_abundance.compute_mass( - z_proxy_limits, mass_proxy_limits - ) - / counts - ) - cluster_masses.append(cluster_mass) - theory_vector_list += cluster_masses - - print(len(theory_vector_list)) - return TheoryVector.from_list(theory_vector_list) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 2beb6491a..ff83e64c5 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -5,7 +5,6 @@ from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping import numpy as np from firecrown.parameters import ParamsMap -from firecrown.integrator import Integrator import numpy.typing as npt # TODO: 1. CCL recomputing and not caching - mass function @@ -16,6 +15,7 @@ # single # cosmology. # * Compute the grid of values for the mass function, then integrate +# Consider emulators for mass function class ClusterAbundance(object): @@ -53,7 +53,6 @@ def __init__( max_z: float, halo_mass_function: pyccl.halos.MassFunc, sky_area: float, - integrator: Integrator, ): self.kernels: List[Kernel] = [] self.halo_mass_function = halo_mass_function @@ -62,7 +61,6 @@ def __init__( self.min_z = min_z self.max_z = max_z self.sky_area = sky_area - self.integrator = integrator self._hmf_cache = {} self._cosmo: Cosmology = None @@ -112,83 +110,23 @@ def mass_function( return return_vals - def get_integrand(self, include_mass=False): + def get_integrand(self, avg_mass=False, avg_redshift=False): def integrand(*int_args): args_map: ArgsMapping = int_args[-1] + z = args_map.get_integral_bounds(int_args, KernelType.z) mass = args_map.get_integral_bounds(int_args, KernelType.mass) + integrand = self.comoving_volume(z) * self.mass_function(mass, z) - if include_mass: + if avg_mass: integrand *= mass + if avg_redshift: + integrand *= z + for kernel in self.kernels: # Think of overhead here, if its worth it - integrand *= ( - kernel.analytic_solution(int_args, args_map) - if kernel.has_analytic_sln - else kernel.distribution(int_args, args_map) - ) + integrand *= kernel.distribution(int_args, args_map) + return integrand return integrand - - def get_integration_bounds(self, z_proxy_limits, mass_proxy_limits): - args_mapping = ArgsMapping() - args_mapping.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - integral_bounds = [(self.min_mass, self.max_mass), (self.min_z, self.max_z)] - extra_args = [] - - for kernel in self.dirac_delta_kernels: - # If any kernel is a dirac delta for z or M, just replace the - # true limits with the proxy limits - if kernel.kernel_type == KernelType.z_proxy: - integral_bounds[1] = z_proxy_limits - elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds[0] = mass_proxy_limits - - mapping_idx = len(args_mapping.integral_bounds.keys()) - for kernel in self.integrable_kernels: - # If any kernel is not a dirac delta, integrate over the relevant limits - args_mapping.integral_bounds[kernel.kernel_type.name] = mapping_idx - mapping_idx += 1 - - match kernel.kernel_type: - case KernelType.z_proxy: - integral_bounds.append(z_proxy_limits) - case KernelType.mass_proxy: - integral_bounds.append(mass_proxy_limits) - case _: - integral_bounds.append(kernel.integral_bounds) - - mapping_idx = 0 - for kernel in self.analytic_kernels: - # Lastly, don't integrate any kernels with an analytic solution - # This means we pass in their limits as extra arguments to the integrator - args_mapping.extra_args[kernel.kernel_type.name] = mapping_idx - mapping_idx += 1 - - match kernel.kernel_type: - case KernelType.z_proxy: - extra_args.append(z_proxy_limits) - case KernelType.mass_proxy: - extra_args.append(mass_proxy_limits) - case _: - extra_args.append(kernel.integral_bounds) - - return integral_bounds, extra_args, args_mapping - - def compute_counts(self, z_proxy_limits, mass_proxy_limits): - bounds, extra_args, args_mapping = self.get_integration_bounds( - z_proxy_limits, mass_proxy_limits - ) - integrand = self.get_integrand() - cc = self.integrator.integrate(integrand, bounds, args_mapping, extra_args) - return cc - - # compute_average(AverageType.shear, etc) - def compute_mass(self, z_proxy_limits, mass_proxy_limits): - bounds, extra_args, args_mapping = self.get_integration_bounds( - z_proxy_limits, mass_proxy_limits - ) - integrand = self.get_integrand(include_mass=True) - cc = self.integrator.integrate(integrand, bounds, args_mapping, extra_args) - return cc diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index b49c1cb88..621c836bb 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -53,9 +53,6 @@ def __init__( def distribution(self, args: List[float], args_map: ArgsMapping): raise NotImplementedError() - def analytic_solution(self, args: List[float], args_map: ArgsMapping): - raise NotImplementedError() - class Completeness(Kernel): def __init__(self): @@ -127,7 +124,7 @@ def observed_value(self, p: Tuple[float, float, float], mass, z): return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - def analytic_solution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgsMapping): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) mass_limits = args_map.get_extra_args(args, self.kernel_type) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index a0fea34a4..c51b2305e 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,5 +1,5 @@ -from firecrown.models.cluster_abundance import ClusterAbundance -from firecrown.models.kernel import Kernel, KernelType +from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.kernel import Kernel, KernelType import pyccl import numpy as np from typing import List, Tuple, Dict From e644ad4da8f9ce0d821e3b140fbc6bd7aa7d5403 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 17 Oct 2023 10:25:53 -0700 Subject: [PATCH 16/80] Switching to Bocquet16 for speed improvements Small fixes. --- .../cluster_number_counts/cluster_redshift_richness.py | 2 +- .../statistic/binned_cluster_number_counts.py | 7 ++++++- firecrown/models/cluster/abundance.py | 10 +--------- firecrown/models/cluster/kernel.py | 7 ++++--- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index a794796f9..df7d9ac2a 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -23,7 +23,7 @@ def get_cluster_abundance(sky_area): - hmf = ccl.halos.MassFuncTinker08() + hmf = ccl.halos.MassFuncBocquet16() min_mass, max_mass = 13.0, 16.0 min_z, max_z = 0.2, 0.8 cluster_abundance = ClusterAbundance( diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 0c404800b..10ed56db1 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -11,6 +11,8 @@ from ....modeling_tools import ModelingTools import numpy as np +import cProfile + class BinnedClusterNumberCounts(Statistic): def __init__( @@ -22,6 +24,7 @@ def __init__( systematics: Optional[List[SourceSystematic]] = None, ): super().__init__() + self.pr = cProfile.Profile() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None self.use_cluster_counts = cluster_counts @@ -72,6 +75,7 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_masses = [] if self.use_cluster_counts or self.use_mean_log_mass: + # self.pr.enable() for z_proxy_limits, mass_proxy_limits in self.bin_limits: bounds, extra_args, args_mapping = self.get_integration_bounds( tools.cluster_abundance, z_proxy_limits, mass_proxy_limits @@ -83,7 +87,8 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: ) cluster_counts.append(counts) - + # self.pr.disable() + # self.pr.dump_stats("profile.prof") theory_vector_list += cluster_counts if self.use_mean_log_mass: diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index ff83e64c5..47e198906 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -7,15 +7,7 @@ from firecrown.parameters import ParamsMap import numpy.typing as npt -# TODO: 1. CCL recomputing and not caching - mass function -# 2. Integrator arguments issues -# * mass_function will ultimately be an NxN grid of values -# This is probably being recomputed for every z, m bin -# * Power spectrum takes the heavy lifting, this will be computed for a -# single -# cosmology. -# * Compute the grid of values for the mass function, then integrate -# Consider emulators for mass function +# TODO: Consider emulators for mass function class ClusterAbundance(object): diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 621c836bb..bdba4033f 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -77,9 +77,9 @@ class Purity(Kernel): def __init__(self): super().__init__(KernelType.purity) - def distribution(self, args: List[float], index_lkp: ArgsMapping): - mass_proxy = args[index_lkp.integral_bounds[KernelType.mass_proxy.name]] - z = args[index_lkp.integral_bounds[KernelType.z.name]] + def distribution(self, args: List[float], args_index_map: ArgsMapping): + mass_proxy = args_index_map.get_integral_bounds(KernelType.mass_proxy) + z = args_index_map.get_integral_bounds(KernelType.z) ln_r = np.log(10**mass_proxy) a_nc = np.log(10) * 0.8612 @@ -128,6 +128,7 @@ def distribution(self, args: List[float], args_map: ArgsMapping): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) mass_limits = args_map.get_extra_args(args, self.kernel_type) + observed_mean_mass = self.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), mass, From c5be8c54ab2d90f6368f3db2eb292e517a5e3b97 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 17 Oct 2023 13:28:18 -0700 Subject: [PATCH 17/80] Adding tests for cluster kernels. --- firecrown/models/cluster/kernel.py | 15 +- tests/test_cluster_data.py | 89 ++++++++++ tests/test_cluster_kernels.py | 265 ++++++++++++++++++++++++----- tests/test_modeling_tools.py | 1 + 4 files changed, 319 insertions(+), 51 deletions(-) create mode 100644 tests/test_cluster_data.py diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index bdba4033f..af3e5c692 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -62,7 +62,6 @@ def distribution(self, args: List[float], args_map: ArgsMapping): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) - # TODO improve parameter names a_nc = 1.1321 b_nc = 0.7751 a_mc = 13.31 @@ -78,17 +77,21 @@ def __init__(self): super().__init__(KernelType.purity) def distribution(self, args: List[float], args_index_map: ArgsMapping): - mass_proxy = args_index_map.get_integral_bounds(KernelType.mass_proxy) - z = args_index_map.get_integral_bounds(KernelType.z) + mass_proxy = args_index_map.get_integral_bounds(args, KernelType.mass_proxy) + z = args_index_map.get_integral_bounds(args, KernelType.z) - ln_r = np.log(10**mass_proxy) a_nc = np.log(10) * 0.8612 b_nc = np.log(10) * 0.3527 a_rc = 2.2183 b_rc = -0.6592 - nc = a_nc + b_nc * (1.0 + z) + + ln_r = np.log(10**mass_proxy) ln_rc = a_rc + b_rc * (1.0 + z) - purity = (ln_r / ln_rc) ** nc / ((ln_r / ln_rc) ** nc + 1.0) + r_over_rc = ln_r / ln_rc + + nc = a_nc + b_nc * (1.0 + z) + + purity = (r_over_rc) ** nc / (r_over_rc**nc + 1.0) return purity diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py new file mode 100644 index 000000000..d5510145e --- /dev/null +++ b/tests/test_cluster_data.py @@ -0,0 +1,89 @@ +import pytest +import numpy as np +from firecrown.models.cluster.abundance_data import AbundanceData +import sacc + + +def test_create_abundance_data_no_survey(): + with pytest.raises( + ValueError, match="The SACC file does not contain the SurveyTracer" + ): + _ = AbundanceData(sacc.Sacc(), "survey", False, False) + + +def test_create_abundance_data_wrong_tracer(): + s = sacc.Sacc() + s.add_tracer("mock", "test", 0.1, 0.2) + with pytest.raises(ValueError, match="The SACC tracer test is not a SurveyTracer"): + _ = AbundanceData(s, "test", False, False) + + +def test_create_abundance_data(): + s = sacc.Sacc() + s.add_tracer("survey", "mock_survey", 4000) + ad = AbundanceData(s, "mock_survey", True, True) + + assert ad.cluster_counts is True + assert ad.mean_log_mass is True + assert ad.survey_nm == "mock_survey" + assert ad.survey_tracer.sky_area == 4000 + assert ad._mass_index == 2 + assert ad._redshift_index == 1 + assert ad._survey_index == 0 + + +def test_validate_tracers(): + s = sacc.Sacc() + s.add_tracer("survey", "mock_survey", 4000) + s.add_tracer("bin_z", "my_tracer", 0, 2) + s.add_data_point( + sacc.standard_types.cluster_counts, ("mock_survey", "my_tracer"), 1 + ) + ad = AbundanceData(s, "mock_survey", False, False) + + tracer_combs = np.array( + s.get_tracer_combinations(sacc.standard_types.cluster_mean_log_mass) + ) + with pytest.raises( + ValueError, + match="The SACC file does not contain any tracers for the" + + f" {sacc.standard_types.cluster_mean_log_mass} data type", + ): + ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_mean_log_mass) + + tracer_combs = np.array( + s.get_tracer_combinations(sacc.standard_types.cluster_counts) + ) + with pytest.raises(ValueError, match="The SACC file must contain 3 tracers"): + ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_counts) + + +def test_filtered_tracers(): + s = sacc.Sacc() + s.add_tracer("survey", "mock_survey", 4000) + s.add_tracer("bin_z", "my_tracer1", 0, 2) + s.add_tracer("bin_z", "my_tracer2", 2, 4) + s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) + s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) + s.add_data_point( + sacc.standard_types.cluster_counts, + ("mock_survey", "my_tracer", "my_other_tracer"), + 1, + ) + s.add_data_point( + sacc.standard_types.cluster_counts, + ("mock_survey", "my_tracer", "my_other_tracer"), + 1, + ) + s.add_data_point( + sacc.standard_types.cluster_mean_log_mass, + ("mock_survey", "my_tracer", "my_other_other_tracer"), + 1, + ) + ad = AbundanceData(s, "mock_survey", False, False) + + filtered_tracers, survey_mask = ad.get_filtered_tracers( + sacc.standard_types.cluster_counts + ) + assert (filtered_tracers == ["mock_survey", "my_tracer", "my_other_tracer"]).all() + assert (survey_mask == [True, False]).all() diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 63d24877f..95399f663 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -1,84 +1,259 @@ -from firecrown.models.kernel import Completeness, Purity, KernelType, Kernel -from firecrown.models.mass_observable import TrueMass, MassRichnessMuSigma -from firecrown.models.redshift import ( - SpectroscopicRedshift, +import pytest +import numpy as np +from firecrown.models.cluster.kernel import ( + Completeness, + Purity, + KernelType, + Kernel, + MassRichnessMuSigma, DESY1PhotometricRedshift, + SpectroscopicRedshift, + TrueMass, + ArgsMapping, ) -def test_desy1_photometric_redshift_kernel(): +def test_create_desy1_photometric_redshift_kernel(): drk = DESY1PhotometricRedshift() assert isinstance(drk, Kernel) assert drk.kernel_type == KernelType.z_proxy - assert drk.is_dirac_delta is True - assert drk.integral_bounds is None - - drk = DESY1PhotometricRedshift([(0, 1)]) assert drk.is_dirac_delta is False - assert len(drk.integral_bounds) == 1 - assert drk.integral_bounds[0] == (0, 1) + assert drk.integral_bounds is None + assert drk.has_analytic_sln is False -def test_spectroscopic_redshift_kernel(): +def test_create_spectroscopic_redshift_kernel(): srk = SpectroscopicRedshift() assert isinstance(srk, Kernel) assert srk.kernel_type == KernelType.z_proxy assert srk.is_dirac_delta is True assert srk.integral_bounds is None - - srk = SpectroscopicRedshift([(0, 1)]) - assert srk.is_dirac_delta is False - assert len(srk.integral_bounds) == 1 - assert srk.integral_bounds[0] == (0, 1) + assert srk.has_analytic_sln is False -def test_musigma_kernel(): +def test_create_musigma_kernel(): msk = MassRichnessMuSigma(1, 1) assert isinstance(msk, Kernel) assert msk.kernel_type == KernelType.mass_proxy - assert msk.is_dirac_delta is True - assert msk.integral_bounds is None - - msk = MassRichnessMuSigma(1, 1, integral_bounds=[(0, 1)]) assert msk.is_dirac_delta is False - assert len(msk.integral_bounds) == 1 - assert msk.integral_bounds[0] == (0, 1) + assert msk.integral_bounds is None + assert msk.has_analytic_sln is True -def test_mass_kernel(): - mk = Mass() +def test_create_mass_kernel(): + mk = TrueMass() assert isinstance(mk, Kernel) - assert mk.kernel_type == KernelType.mass + assert mk.kernel_type == KernelType.mass_proxy assert mk.is_dirac_delta is True assert mk.integral_bounds is None - - mk = Mass([(0, 1)]) - assert mk.is_dirac_delta is False - assert len(mk.integral_bounds) == 1 - assert mk.integral_bounds[0] == (0, 1) + assert mk.has_analytic_sln is False -def test_completeness_kernel(): +def test_create_completeness_kernel(): ck = Completeness() assert isinstance(ck, Kernel) assert ck.kernel_type == KernelType.completeness - assert ck.is_dirac_delta is True - assert ck.integral_bounds is None - - ck = Completeness([(0, 1)]) assert ck.is_dirac_delta is False - assert len(ck.integral_bounds) == 1 - assert ck.integral_bounds[0] == (0, 1) + assert ck.integral_bounds is None + assert ck.has_analytic_sln is False -def test_purity_kernel(): +def test_create_purity_kernel(): pk = Purity() assert isinstance(pk, Kernel) assert pk.kernel_type == KernelType.purity - assert pk.is_dirac_delta is True + assert pk.is_dirac_delta is False assert pk.integral_bounds is None + assert pk.has_analytic_sln is False - pk = Purity([(0, 1)]) - assert pk.is_dirac_delta is False - assert len(pk.integral_bounds) == 1 - assert pk.integral_bounds[0] == (0, 1) + +def test_spec_z_distribution(): + srk = SpectroscopicRedshift() + assert srk.distribution([0.5], ArgsMapping()) == 1.0 + + +def test_true_mass_distribution(): + tmk = TrueMass() + assert tmk.distribution([0.5], ArgsMapping()) == 1.0 + + +def test_purity_distribution(): + pk = Purity() + mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + + arguments = np.array(list(zip(mass_proxy, z))) + map = ArgsMapping() + map.integral_bounds = {KernelType.mass_proxy.name: 0, KernelType.z.name: 1} + + truth = np.array( + [ + 0.77657274, + 0.96966127, + 0.99286409, + 0.99780586, + 0.999224, + 0.99970302, + 0.99988111, + 0.99995125, + 0.99997982, + 0.99999166, + ] + ) + + purity = pk.distribution([arguments], map) + for ref, true in zip(purity, truth): + assert ref == pytest.approx(true, rel=1e-7, abs=0.0) + + +def test_completeness_distribution(): + ck = Completeness() + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + + bounds = np.array(list(zip(mass, z))) + map = ArgsMapping() + map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + truth = np.array( + [ + 0.0056502277493542, + 0.01896566878380423, + 0.03805597500308377, + 0.06224888967250564, + 0.09124569979282898, + 0.12486247682690908, + 0.16290218589569144, + 0.20507815091349266, + 0.2509673905442634, + 0.2999886170051561, + ] + ) + + comp = ck.distribution([bounds], map) + for ref, true in zip(comp, truth): + assert ref == pytest.approx(true, rel=1e-7, abs=0.0) + + +def test_des_photoz_kernel_distribution(): + dpk = DESY1PhotometricRedshift() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) + + bounds = np.array(list(zip(mass, z, z_proxy))) + + map = ArgsMapping() + map.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + } + + truth = [ + 7.134588921656481, + 6.557328601698999, + 6.065367634804254, + 5.641316284718016, + 5.272157878477569, + 4.9479710868093685, + 4.661070179674804, + 4.405413986167644, + 4.176191421334415, + 3.969525474770118, + ] + + spread = dpk.distribution([bounds], map) + for ref, true in zip(spread, truth): + assert ref == pytest.approx(true, rel=1e-7, abs=0.0) + + +# class ArgsMapping: +# def __init__(self): +# self.integral_bounds = dict() +# self.extra_args = dict() + +# self.integral_bounds_idx = 0 +# self.extra_args_idx = 1 + +# def get_integral_bounds(self, int_args, kernel_type: KernelType): +# bounds_values = int_args[self.integral_bounds_idx] +# return bounds_values[:, self.integral_bounds[kernel_type.name]] + +# def get_extra_args(self, int_args, kernel_type: KernelType): +# extra_values = int_args[self.extra_args_idx] +# return extra_values[self.extra_args[kernel_type.name]] + + +def test_args_mapper_extra_args(): + args_map = ArgsMapping() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + extra_args = [(1, 2, 3), ("hello world")] + args_map.extra_args = {KernelType.mass_proxy.name: 0, KernelType.z_proxy.name: 1} + + integral_bounds = np.array(list(zip(mass, z))) + int_args = [integral_bounds, extra_args] + + assert args_map.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) + assert args_map.get_extra_args(int_args, KernelType.z_proxy) == "hello world" + + +def test_args_mapper_integral_bounds(): + args_map = ArgsMapping() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) + mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) + + args_map.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + } + + integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) + int_args = [integral_bounds] + + assert (mass == args_map.get_integral_bounds(int_args, KernelType.mass)).all() + assert (z == args_map.get_integral_bounds(int_args, KernelType.z)).all() + assert (z_proxy == args_map.get_integral_bounds(int_args, KernelType.z_proxy)).all() + assert ( + mass_proxy == args_map.get_integral_bounds(int_args, KernelType.mass_proxy) + ).all() + + +def test_create_args_mapping(): + am = ArgsMapping() + assert am.integral_bounds == dict() + assert am.extra_args == dict() + assert am.integral_bounds_idx == 0 + assert am.extra_args_idx == 1 + + +# int_args = [ +# (values im integrating over), +# (extra arguments for my integrand), +# args_mapper = { +# integral_bounds = { +# mass at index 0 +# z at index 1 +# z_proxy at index 2 +# } +# extra_args = { +# mass_proxy at index 0 +# } +# } +# ] + +# models/Updatable +# firecrown/ +# cluster_abundance/ +# cluster_abundance/ +# firecrown/ diff --git a/tests/test_modeling_tools.py b/tests/test_modeling_tools.py index bd17ff7e3..8dd33bd2d 100644 --- a/tests/test_modeling_tools.py +++ b/tests/test_modeling_tools.py @@ -18,6 +18,7 @@ def test_default_constructed_state(): # Default constructed state is pretty barren... assert tools.ccl_cosmo is None assert tools.pt_calculator is None + assert tools.cluster_abundance is None assert len(tools.powerspectra) == 0 From 670296a26326be893fb0ab4b0e0cc2aa5dfd6cca Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 17 Oct 2023 15:22:40 -0700 Subject: [PATCH 18/80] Added unit tests for abundance and abundance data classes. --- tests/test_cluster_abundance.py | 118 +++++++++++++++++--------------- tests/test_cluster_data.py | 80 ++++++++++++++-------- 2 files changed, 112 insertions(+), 86 deletions(-) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index c51b2305e..31300d382 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,79 +1,83 @@ +import pytest from firecrown.models.cluster.abundance import ClusterAbundance -from firecrown.models.cluster.kernel import Kernel, KernelType +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping import pyccl import numpy as np -from typing import List, Tuple, Dict +from typing import List, Tuple +from firecrown.parameters import ParamsMap -class MockKernel(Kernel): - def __init__(self, integral_bounds: List[Tuple[float, float]] = None): - super().__init__(KernelType.mass, integral_bounds) +@pytest.fixture() +def cl_abundance(): + hmf = pyccl.halos.MassFuncBocquet16() + ca = ClusterAbundance(13, 17, 0, 2, hmf, 360.0**2) + return ca - def distribution(self, args: List[float], index_lkp: Dict[str, int]): - return 1.0 +class MockKernel(Kernel): + def __init__( + self, + kernel_type: KernelType, + is_dirac_delta: bool = False, + has_analytic_sln: bool = False, + integral_bounds: List[Tuple[float, float]] = None, + ): + super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) + + def distribution(self, args: List[float], args_map: ArgsMapping): + return 1.0 -def test_cluster_abundance_init(): - hmf = pyccl.halos.MassFuncTinker08() - ca = ClusterAbundance(hmf) - assert ca is not None - assert ca.cosmo is None - assert isinstance(ca.halo_mass_function, pyccl.halos.MassFuncTinker08) +def test_cluster_abundance_init(cl_abundance: ClusterAbundance): + assert cl_abundance is not None + assert cl_abundance.cosmo is None + assert isinstance(cl_abundance.halo_mass_function, pyccl.halos.MassFuncBocquet16) + assert cl_abundance.min_mass == 13.0 + assert cl_abundance.max_mass == 17.0 + assert cl_abundance.min_z == 0.0 + assert cl_abundance.max_z == 2.0 + assert cl_abundance.sky_area == 360.0**2 + assert cl_abundance.sky_area_rad == 4 * np.pi**2 + assert len(cl_abundance.kernels) == 0 -def test_cluster_update_ingredients(): - hmf = pyccl.halos.MassFuncTinker08() - ca = ClusterAbundance(hmf) - assert ca.cosmo is None +def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): + assert cl_abundance.cosmo is None cosmo = pyccl.CosmologyVanillaLCDM() + pmap = ParamsMap() - ca.update_ingredients(cosmo) - assert ca.cosmo is not None - assert ca.cosmo == cosmo - - -def test_cluster_sky_area(): - hmf = pyccl.halos.MassFuncTinker08() - ca = ClusterAbundance(hmf) - assert ca.sky_area == 360.0**2 - assert ca.sky_area_rad == 4 * np.pi**2 - - ca.sky_area = 180.0**2 - assert ca.sky_area == 180.0**2 - assert ca.sky_area_rad == np.pi**2 + cl_abundance.update_ingredients(cosmo, pmap) + assert cl_abundance.cosmo is not None + assert cl_abundance.cosmo == cosmo -def test_cluster_add_kernel(): - hmf = pyccl.halos.MassFuncTinker08() - ca = ClusterAbundance(hmf) - assert len(ca.kernels) == 0 +def test_cluster_sky_area(cl_abundance: ClusterAbundance): + assert cl_abundance.sky_area == 360.0**2 + assert cl_abundance.sky_area_rad == 4 * np.pi**2 - ca.add_kernel(MockKernel()) - assert len(ca.kernels) == 1 - assert isinstance(ca.kernels[0], Kernel) + cl_abundance.sky_area = 180.0**2 + assert cl_abundance.sky_area == 180.0**2 + assert cl_abundance.sky_area_rad == np.pi**2 -def test_cluster_abundance_bounds(): - hmf = pyccl.halos.MassFuncTinker08() - ca = ClusterAbundance(hmf) - bounds, index_lookup = ca.get_integration_bounds() - assert len(bounds) == 0 - assert len(index_lookup) == 0 +def test_cluster_add_kernel(cl_abundance: ClusterAbundance): + assert len(cl_abundance.kernels) == 0 - ca.add_kernel(MockKernel()) - bounds, index_lookup = ca.get_integration_bounds() - assert len(bounds) == 0 - assert len(index_lookup) == 0 + cl_abundance.add_kernel(MockKernel(KernelType.mass)) + assert len(cl_abundance.kernels) == 1 + assert isinstance(cl_abundance.kernels[0], Kernel) + assert cl_abundance.kernels[0].kernel_type == KernelType.mass - ca.add_kernel(MockKernel([(0, 1)])) - bounds, index_lookup = ca.get_integration_bounds() - assert len(bounds) == 1 - assert len(index_lookup) == 1 - assert index_lookup["mass"] == 0 - assert bounds[index_lookup["mass"]] == [(0, 1)] + assert len(cl_abundance.analytic_kernels) == 0 + assert len(cl_abundance.dirac_delta_kernels) == 0 + assert len(cl_abundance.integrable_kernels) == 1 + cl_abundance.add_kernel(MockKernel(KernelType.mass, True)) + assert len(cl_abundance.analytic_kernels) == 0 + assert len(cl_abundance.dirac_delta_kernels) == 1 + assert len(cl_abundance.integrable_kernels) == 1 -# def test_cluster_abundance_integrand(): -# hmf = pyccl.halos.MassFuncTinker08() -# ca = ClusterAbundance(hmf) + cl_abundance.add_kernel(MockKernel(KernelType.mass, False, True)) + assert len(cl_abundance.analytic_kernels) == 1 + assert len(cl_abundance.dirac_delta_kernels) == 1 + assert len(cl_abundance.integrable_kernels) == 1 diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index d5510145e..b60099c24 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -4,6 +4,31 @@ import sacc +@pytest.fixture +def complicated_sacc_data(): + cc = sacc.standard_types.cluster_counts + mlm = sacc.standard_types.cluster_mean_log_mass + + s = sacc.Sacc() + s.add_tracer("survey", "my_survey", 4000) + s.add_tracer("survey", "not_my_survey", 4000) + s.add_tracer("bin_z", "my_tracer1", 0, 2) + s.add_tracer("bin_z", "my_tracer2", 2, 4) + s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) + s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) + + s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) + s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) + s.add_data_point(cc, ("not_my_survey", "my_tracer1", "my_other_tracer2"), 1) + + s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer2"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) + + return s + + def test_create_abundance_data_no_survey(): with pytest.raises( ValueError, match="The SACC file does not contain the SurveyTracer" @@ -13,7 +38,7 @@ def test_create_abundance_data_no_survey(): def test_create_abundance_data_wrong_tracer(): s = sacc.Sacc() - s.add_tracer("mock", "test", 0.1, 0.2) + s.add_tracer("bin_richness", "test", 0.1, 0.2) with pytest.raises(ValueError, match="The SACC tracer test is not a SurveyTracer"): _ = AbundanceData(s, "test", False, False) @@ -58,32 +83,29 @@ def test_validate_tracers(): ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_counts) -def test_filtered_tracers(): - s = sacc.Sacc() - s.add_tracer("survey", "mock_survey", 4000) - s.add_tracer("bin_z", "my_tracer1", 0, 2) - s.add_tracer("bin_z", "my_tracer2", 2, 4) - s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) - s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) - s.add_data_point( - sacc.standard_types.cluster_counts, - ("mock_survey", "my_tracer", "my_other_tracer"), - 1, - ) - s.add_data_point( - sacc.standard_types.cluster_counts, - ("mock_survey", "my_tracer", "my_other_tracer"), - 1, - ) - s.add_data_point( - sacc.standard_types.cluster_mean_log_mass, - ("mock_survey", "my_tracer", "my_other_other_tracer"), - 1, - ) - ad = AbundanceData(s, "mock_survey", False, False) +def test_filtered_tracers(complicated_sacc_data): + ad = AbundanceData(complicated_sacc_data, "my_survey", False, False) + cc = sacc.standard_types.cluster_counts + filtered_tracers, survey_mask = ad.get_filtered_tracers(cc) + my_tracers = [ + ("my_survey", "my_tracer1", "my_other_tracer1"), + ("my_survey", "my_tracer1", "my_other_tracer2"), + ] + assert (filtered_tracers == my_tracers).all() + assert (survey_mask == [True, True, False]).all() - filtered_tracers, survey_mask = ad.get_filtered_tracers( - sacc.standard_types.cluster_counts - ) - assert (filtered_tracers == ["mock_survey", "my_tracer", "my_other_tracer"]).all() - assert (survey_mask == [True, False]).all() + +def test_get_data_and_indices(complicated_sacc_data): + ad = AbundanceData(complicated_sacc_data, "my_survey", False, False) + cc = sacc.standard_types.cluster_counts + data, indices = ad.get_data_and_indices(cc) + + assert data == [1, 1] + assert indices == [0, 1] + + +def test_get_bin_limits(complicated_sacc_data): + ad = AbundanceData(complicated_sacc_data, "my_survey", False, False) + cc = sacc.standard_types.cluster_counts + limits = ad.get_bin_limits(cc) + assert limits == [[(0, 2), (0, 2)], [(0, 2), (2, 4)]] From 807b199cba7dcbd28c3d62d1cdefe727a2d92835 Mon Sep 17 00:00:00 2001 From: m-aguena Date: Wed, 18 Oct 2023 03:25:37 -0700 Subject: [PATCH 19/80] mv __init__ to murata --- firecrown/models/cluster/kernel.py | 68 ----------------- .../models/cluster/mass_proxy/__init__.py | 0 .../models/cluster/mass_proxy/gaussian.py | 60 +++++++++++++++ firecrown/models/cluster/mass_proxy/murata.py | 75 +++++++++++++++++++ 4 files changed, 135 insertions(+), 68 deletions(-) create mode 100644 firecrown/models/cluster/mass_proxy/__init__.py create mode 100644 firecrown/models/cluster/mass_proxy/gaussian.py create mode 100644 firecrown/models/cluster/mass_proxy/murata.py diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index af3e5c692..c5a68c999 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -9,7 +9,6 @@ from firecrown.updatable import Updatable import pdb - class KernelType(Enum): mass = 1 z = 2 @@ -95,73 +94,6 @@ def distribution(self, args: List[float], args_index_map: ArgsMapping): return purity -class MassRichnessMuSigma(Kernel): - def __init__( - self, - pivot_mass, - pivot_redshift, - integral_bounds: List[Tuple[float, float]] = None, - ): - super().__init__(KernelType.mass_proxy, False, True, integral_bounds) - self.pivot_mass = pivot_mass - self.pivot_redshift = pivot_redshift - self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) - self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) - - # Updatable parameters - self.mu_p0 = parameters.create() - self.mu_p1 = parameters.create() - self.mu_p2 = parameters.create() - self.sigma_p0 = parameters.create() - self.sigma_p1 = parameters.create() - self.sigma_p2 = parameters.create() - - # Verify this gets called last or first - - def observed_value(self, p: Tuple[float, float, float], mass, z): - """Return observed quantity corrected by redshift and mass.""" - - ln_mass = mass * np.log(10) - delta_ln_mass = ln_mass - self.pivot_mass - delta_z = np.log1p(z) - self.log1p_pivot_redshift - - return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - - def distribution(self, args: List[float], args_map: ArgsMapping): - mass = args_map.get_integral_bounds(args, KernelType.mass) - z = args_map.get_integral_bounds(args, KernelType.z) - mass_limits = args_map.get_extra_args(args, self.kernel_type) - - observed_mean_mass = self.observed_value( - (self.mu_p0, self.mu_p1, self.mu_p2), - mass, - z, - ) - observed_mass_sigma = self.observed_value( - (self.sigma_p0, self.sigma_p1, self.sigma_p2), - mass, - z, - ) - - x_min = (observed_mean_mass - mass_limits[0] * np.log(10.0)) / ( - np.sqrt(2.0) * observed_mass_sigma - ) - x_max = (observed_mean_mass - mass_limits[1] * np.log(10.0)) / ( - np.sqrt(2.0) * observed_mass_sigma - ) - - return_vals = np.empty_like(x_min) - mask1 = (x_max > 3.0) | (x_min < -3.0) - mask2 = ~mask1 - - return_vals[mask1] = ( - -(special.erfc(x_min[mask1]) - special.erfc(x_max[mask1])) / 2.0 - ) - return_vals[mask2] = ( - special.erf(x_min[mask2]) - special.erf(x_max[mask2]) - ) / 2.0 - - return return_vals class TrueMass(Kernel): diff --git a/firecrown/models/cluster/mass_proxy/__init__.py b/firecrown/models/cluster/mass_proxy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/firecrown/models/cluster/mass_proxy/gaussian.py b/firecrown/models/cluster/mass_proxy/gaussian.py new file mode 100644 index 000000000..a26bef8e6 --- /dev/null +++ b/firecrown/models/cluster/mass_proxy/gaussian.py @@ -0,0 +1,60 @@ +from typing import List + +import numpy as np +from scipy import special + +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping + + +class MassRichnessGaussian(Kernel): + def _init_params(self): + """Instanciate parameters of mass richenss distribution.""" + return NotImplementedError + + def get_proxy_mean(self, mass, z): + """Return observed quantity corrected by redshift and mass.""" + return NotImplementedError + + def get_proxy_sigma(self, mass, z): + """Return observed scatter corrected by redshift and mass.""" + return NotImplementedError + + def _distribution_binned(self, args: List[float], args_map: ArgsMapping): + mass = args_map.get_integral_bounds(args, KernelType.mass) + z = args_map.get_integral_bounds(args, KernelType.z) + mass_proxy_limits = args_map.get_extra_args(args, self.kernel_type) + + proxy_mean = self.get_proxy_mean(mass, z) + proxy_sigma = self.get_proxy_sigma(mass, z) + + x_min = (proxy_mean - mass_proxy_limits[0] * np.log(10.0)) / ( + np.sqrt(2.0) * proxy_sigma + ) + x_max = (proxy_mean - mass_proxy_limits[1] * np.log(10.0)) / ( + np.sqrt(2.0) * proxy_sigma + ) + + return_vals = np.empty_like(x_min) + mask1 = (x_max > 3.0) | (x_min < -3.0) + mask2 = ~mask1 + + return_vals[mask1] = ( + -(special.erfc(x_min[mask1]) - special.erfc(x_max[mask1])) / 2.0 + ) + return_vals[mask2] = ( + special.erf(x_min[mask2]) - special.erf(x_max[mask2]) + ) / 2.0 + + return return_vals + + def _distribution_unbinned(self, args: List[float], args_map: ArgsMapping): + mass = args_map.get_integral_bounds(args, KernelType.mass) + z = args_map.get_integral_bounds(args, KernelType.z) + mass_proxy = args_map.get_extra_args(args, self.kernel_type) + + proxy_mean = self.get_proxy_mean(mass, z) + proxy_sigma = self.get_proxy_sigma(mass, z) + + return np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( + 2 * np.pi * proxy_sigma + ) diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py new file mode 100644 index 000000000..89c12ec7d --- /dev/null +++ b/firecrown/models/cluster/mass_proxy/murata.py @@ -0,0 +1,75 @@ +from typing import List, Tuple + +import numpy as np + +from firecrown import parameters +from firecrown.models.cluster.kernel import KernelType +from firecrown.models.cluster.mass_proxy.gaussian import MassRichnessGaussian + + +def _observed_value( + p: Tuple[float, float, float], mass, z, pivot_mass, log1p_pivot_redshift +): + """Return observed quantity corrected by redshift and mass.""" + + ln_mass = mass * np.log(10) + delta_ln_mass = ln_mass - pivot_mass + delta_z = np.log1p(z) - log1p_pivot_redshift + + return p[0] + p[1] * delta_ln_mass + p[2] * delta_z + + +class MassRichnessCore(MassRichnessGaussian): + def __init__( + self, + pivot_mass, + pivot_redshift, + integral_bounds: List[Tuple[float, float]] = None, + ): + super().__init__(KernelType.mass_proxy, False, True, integral_bounds) + + self.pivot_mass = pivot_mass + self.pivot_redshift = pivot_redshift + self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) + self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) + + # Updatable parameters + self.mu_p0 = parameters.create() + self.mu_p1 = parameters.create() + self.mu_p2 = parameters.create() + self.sigma_p0 = parameters.create() + self.sigma_p1 = parameters.create() + self.sigma_p2 = parameters.create() + + # Verify this gets called last or first + + def get_proxy_mean(self, mass, z): + """Return observed quantity corrected by redshift and mass.""" + return _observed_value( + (self.mu_p0, self.mu_p1, self.mu_p2), + mass, + z, + self.pivot_mass, + self.log1p_pivot_redshift, + ) + + def get_proxy_sigma(self, mass, z): + """Return observed scatter corrected by redshift and mass.""" + return _observed_value( + (self.sigma_p0, self.sigma_p1, self.sigma_p2), + mass, + z, + self.pivot_mass, + self.log1p_pivot_redshift, + ) + + +### used to be MassRichnessMuSigma ### +class MassRichnessBinned(MassRichnessCore): + def distribution(self, args, args_map): + return self._distribution_binned(args, args_map) + + +class MassRichnessUnbinned(MassRichnessCore): + def distribution(self, args, args_map): + return self._distribution_unbinned(args, args_map) From e95488d3dee72a0f6dd19eeb955fd7c73351b965 Mon Sep 17 00:00:00 2001 From: m-aguena Date: Wed, 18 Oct 2023 03:26:15 -0700 Subject: [PATCH 20/80] rm leftover code --- firecrown/models/cluster/mass_proxy/gaussian.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/firecrown/models/cluster/mass_proxy/gaussian.py b/firecrown/models/cluster/mass_proxy/gaussian.py index a26bef8e6..02ce162c9 100644 --- a/firecrown/models/cluster/mass_proxy/gaussian.py +++ b/firecrown/models/cluster/mass_proxy/gaussian.py @@ -7,10 +7,6 @@ class MassRichnessGaussian(Kernel): - def _init_params(self): - """Instanciate parameters of mass richenss distribution.""" - return NotImplementedError - def get_proxy_mean(self, mass, z): """Return observed quantity corrected by redshift and mass.""" return NotImplementedError From 513b6225fe90f552aaf67499dcfd240726522d7c Mon Sep 17 00:00:00 2001 From: m-aguena Date: Wed, 18 Oct 2023 10:03:16 -0700 Subject: [PATCH 21/80] update example --- examples/cluster_number_counts/cluster_redshift_richness.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index df7d9ac2a..ce916af60 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -16,10 +16,10 @@ from firecrown.models.cluster.kernel import ( Completeness, DESY1PhotometricRedshift, - MassRichnessMuSigma, Purity, SpectroscopicRedshift, ) +from firecrown.models.cluster.kernel.mass_proxy.murata import MassRichnessBinned def get_cluster_abundance(sky_area): @@ -32,7 +32,7 @@ def get_cluster_abundance(sky_area): # Create and add the kernels you want in your cluster abundance pivot_mass, pivot_redshift = 14.625862906, 0.6 - mass_observable_kernel = MassRichnessMuSigma(pivot_mass, pivot_redshift) + mass_observable_kernel = MassRichnessBinned(pivot_mass, pivot_redshift) cluster_abundance.add_kernel(mass_observable_kernel) redshift_proxy_kernel = SpectroscopicRedshift() From c82fb9d2a316306ca5771012f268382eec7d8670 Mon Sep 17 00:00:00 2001 From: m-aguena Date: Wed, 18 Oct 2023 10:28:40 -0700 Subject: [PATCH 22/80] rename Murata classes --- examples/cluster_number_counts/cluster_redshift_richness.py | 4 ++-- firecrown/models/cluster/mass_proxy/murata.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index ce916af60..c6d1e6fda 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -19,7 +19,7 @@ Purity, SpectroscopicRedshift, ) -from firecrown.models.cluster.kernel.mass_proxy.murata import MassRichnessBinned +from firecrown.models.cluster.kernel.mass_proxy.murata import MurataBinned def get_cluster_abundance(sky_area): @@ -32,7 +32,7 @@ def get_cluster_abundance(sky_area): # Create and add the kernels you want in your cluster abundance pivot_mass, pivot_redshift = 14.625862906, 0.6 - mass_observable_kernel = MassRichnessBinned(pivot_mass, pivot_redshift) + mass_observable_kernel = MurataBinned(pivot_mass, pivot_redshift) cluster_abundance.add_kernel(mass_observable_kernel) redshift_proxy_kernel = SpectroscopicRedshift() diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py index 89c12ec7d..39eed81fc 100644 --- a/firecrown/models/cluster/mass_proxy/murata.py +++ b/firecrown/models/cluster/mass_proxy/murata.py @@ -19,7 +19,7 @@ def _observed_value( return p[0] + p[1] * delta_ln_mass + p[2] * delta_z -class MassRichnessCore(MassRichnessGaussian): +class MurataCore(MassRichnessGaussian): def __init__( self, pivot_mass, @@ -65,11 +65,11 @@ def get_proxy_sigma(self, mass, z): ### used to be MassRichnessMuSigma ### -class MassRichnessBinned(MassRichnessCore): +class MurataBinned(MurataCore): def distribution(self, args, args_map): return self._distribution_binned(args, args_map) -class MassRichnessUnbinned(MassRichnessCore): +class MurataUnbinned(MurataCore): def distribution(self, args, args_map): return self._distribution_unbinned(args, args_map) From 2fa9fa71c05b57c2727d72c489abda82b2dbfe6c Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 18 Oct 2023 13:53:55 -0700 Subject: [PATCH 23/80] Fixing a small bug in the new Murata class Adding unit tests for the new Murata classes --- firecrown/models/cluster/kernel.py | 6 +- firecrown/models/cluster/mass_proxy/murata.py | 3 +- tests/test_cluster_args_mapping.py | 56 ++++++++ tests/test_cluster_kernels.py | 100 -------------- tests/test_cluster_mass_richness.py | 130 ++++++++++++++++++ 5 files changed, 188 insertions(+), 107 deletions(-) create mode 100644 tests/test_cluster_args_mapping.py create mode 100644 tests/test_cluster_mass_richness.py diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index c5a68c999..88b32b3fa 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -3,12 +3,10 @@ from typing import List, Tuple import numpy as np -from scipy import special - -from firecrown import parameters from firecrown.updatable import Updatable import pdb + class KernelType(Enum): mass = 1 z = 2 @@ -94,8 +92,6 @@ def distribution(self, args: List[float], args_index_map: ArgsMapping): return purity - - class TrueMass(Kernel): def __init__(self): super().__init__(KernelType.mass_proxy, True) diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py index 39eed81fc..ca5a05552 100644 --- a/firecrown/models/cluster/mass_proxy/murata.py +++ b/firecrown/models/cluster/mass_proxy/murata.py @@ -28,9 +28,8 @@ def __init__( ): super().__init__(KernelType.mass_proxy, False, True, integral_bounds) - self.pivot_mass = pivot_mass self.pivot_redshift = pivot_redshift - self.pivot_mass = self.pivot_mass * np.log(10.0) # ln(M) + self.pivot_mass = pivot_mass * np.log(10.0) # ln(M) self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) # Updatable parameters diff --git a/tests/test_cluster_args_mapping.py b/tests/test_cluster_args_mapping.py new file mode 100644 index 000000000..8a26b2e70 --- /dev/null +++ b/tests/test_cluster_args_mapping.py @@ -0,0 +1,56 @@ +import numpy as np +from firecrown.models.cluster.kernel import ( + KernelType, + ArgsMapping, +) + + +def test_args_mapper_extra_args(): + args_map = ArgsMapping() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + extra_args = [(1, 2, 3), ("hello world")] + args_map.extra_args = {KernelType.mass_proxy.name: 0, KernelType.z_proxy.name: 1} + + integral_bounds = np.array(list(zip(mass, z))) + int_args = [integral_bounds, extra_args] + + assert args_map.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) + assert args_map.get_extra_args(int_args, KernelType.z_proxy) == "hello world" + + +def test_args_mapper_integral_bounds(): + args_map = ArgsMapping() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) + mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) + + args_map.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + } + + integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) + int_args = [integral_bounds] + + assert (mass == args_map.get_integral_bounds(int_args, KernelType.mass)).all() + assert (z == args_map.get_integral_bounds(int_args, KernelType.z)).all() + assert (z_proxy == args_map.get_integral_bounds(int_args, KernelType.z_proxy)).all() + assert ( + mass_proxy == args_map.get_integral_bounds(int_args, KernelType.mass_proxy) + ).all() + + +def test_create_args_mapping(): + am = ArgsMapping() + assert am.integral_bounds == dict() + assert am.extra_args == dict() + assert am.integral_bounds_idx == 0 + assert am.extra_args_idx == 1 diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 95399f663..a5840ad58 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -5,7 +5,6 @@ Purity, KernelType, Kernel, - MassRichnessMuSigma, DESY1PhotometricRedshift, SpectroscopicRedshift, TrueMass, @@ -31,15 +30,6 @@ def test_create_spectroscopic_redshift_kernel(): assert srk.has_analytic_sln is False -def test_create_musigma_kernel(): - msk = MassRichnessMuSigma(1, 1) - assert isinstance(msk, Kernel) - assert msk.kernel_type == KernelType.mass_proxy - assert msk.is_dirac_delta is False - assert msk.integral_bounds is None - assert msk.has_analytic_sln is True - - def test_create_mass_kernel(): mk = TrueMass() assert isinstance(mk, Kernel) @@ -167,93 +157,3 @@ def test_des_photoz_kernel_distribution(): spread = dpk.distribution([bounds], map) for ref, true in zip(spread, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) - - -# class ArgsMapping: -# def __init__(self): -# self.integral_bounds = dict() -# self.extra_args = dict() - -# self.integral_bounds_idx = 0 -# self.extra_args_idx = 1 - -# def get_integral_bounds(self, int_args, kernel_type: KernelType): -# bounds_values = int_args[self.integral_bounds_idx] -# return bounds_values[:, self.integral_bounds[kernel_type.name]] - -# def get_extra_args(self, int_args, kernel_type: KernelType): -# extra_values = int_args[self.extra_args_idx] -# return extra_values[self.extra_args[kernel_type.name]] - - -def test_args_mapper_extra_args(): - args_map = ArgsMapping() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - - extra_args = [(1, 2, 3), ("hello world")] - args_map.extra_args = {KernelType.mass_proxy.name: 0, KernelType.z_proxy.name: 1} - - integral_bounds = np.array(list(zip(mass, z))) - int_args = [integral_bounds, extra_args] - - assert args_map.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) - assert args_map.get_extra_args(int_args, KernelType.z_proxy) == "hello world" - - -def test_args_mapper_integral_bounds(): - args_map = ArgsMapping() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) - mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) - - args_map.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - } - - integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) - int_args = [integral_bounds] - - assert (mass == args_map.get_integral_bounds(int_args, KernelType.mass)).all() - assert (z == args_map.get_integral_bounds(int_args, KernelType.z)).all() - assert (z_proxy == args_map.get_integral_bounds(int_args, KernelType.z_proxy)).all() - assert ( - mass_proxy == args_map.get_integral_bounds(int_args, KernelType.mass_proxy) - ).all() - - -def test_create_args_mapping(): - am = ArgsMapping() - assert am.integral_bounds == dict() - assert am.extra_args == dict() - assert am.integral_bounds_idx == 0 - assert am.extra_args_idx == 1 - - -# int_args = [ -# (values im integrating over), -# (extra arguments for my integrand), -# args_mapper = { -# integral_bounds = { -# mass at index 0 -# z at index 1 -# z_proxy at index 2 -# } -# extra_args = { -# mass_proxy at index 0 -# } -# } -# ] - -# models/Updatable -# firecrown/ -# cluster_abundance/ -# cluster_abundance/ -# firecrown/ diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py new file mode 100644 index 000000000..e5b064bb3 --- /dev/null +++ b/tests/test_cluster_mass_richness.py @@ -0,0 +1,130 @@ +import pytest +import numpy as np +from firecrown.models.cluster.mass_proxy.murata import MurataBinned, _observed_value +from firecrown.models.cluster.kernel import ( + KernelType, + Kernel, + ArgsMapping, +) + +PIVOT_Z = 0.6 +PIVOT_MASS = 14.625862906 + + +@pytest.fixture(name="murata_binned_relation") +def fixture_cluster_mass_rich() -> MurataBinned: + """Initialize cluster object.""" + + mr = MurataBinned(PIVOT_MASS, PIVOT_Z) + + # Set the parameters to the values used in the test + # they should be such that the variance is always positive. + mr.mu_p0 = 3.00 + mr.mu_p1 = 0.086 + mr.mu_p2 = 0.01 + mr.sigma_p0 = 3.0 + mr.sigma_p1 = 0.07 + mr.sigma_p2 = 0.01 + + return mr + + +def test_create_musigma_kernel(): + mb = MurataBinned(1, 1) + assert isinstance(mb, Kernel) + assert mb.kernel_type == KernelType.mass_proxy + assert mb.is_dirac_delta is False + assert mb.integral_bounds is None + assert mb.has_analytic_sln is True + assert mb.pivot_mass == 1 * np.log(10) + assert mb.pivot_redshift == 1 + assert mb.log1p_pivot_redshift == np.log1p(1) + + assert mb.mu_p0 is None + assert mb.mu_p1 is None + assert mb.mu_p2 is None + assert mb.sigma_p0 is None + assert mb.sigma_p1 is None + assert mb.sigma_p2 is None + + +def test_cluster_observed_z(): + for z in np.geomspace(1.0e-18, 2.0, 20): + f_z = _observed_value((0.0, 0.0, 1.0), 0.0, z, 0, 0) + assert f_z == pytest.approx(np.log1p(z), 1.0e-7, 0.0) + + +def test_cluster_observed_mass(): + for logM in np.linspace(10.0, 16.0, 20): + f_logM = _observed_value((0.0, 1.0, 0.0), logM, 0.0, 0, 0) + + assert f_logM == pytest.approx(logM * np.log(10.0), 1.0e-7, 0.0) + + +def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned): + logM_array = np.linspace(7.0, 26.0, 2000) + for z in np.geomspace(1.0e-18, 2.0, 20): + flip = False + for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): + extra_args = [(1.0, 5.0)] + + args1 = [np.array([[logM_0, z]]), extra_args] + args2 = [np.array([[logM_1, z]]), extra_args] + + args_map = ArgsMapping() + args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + args_map.extra_args = {KernelType.mass_proxy.name: 0} + + probability_0 = murata_binned_relation.distribution(args1, args_map) + probability_1 = murata_binned_relation.distribution(args2, args_map) + + assert probability_0 >= 0 + assert probability_1 >= 0 + + # Probability should be initially monotonically increasing + # and then monotonically decreasing. It should flip only once. + + # Test for the flip + if (not flip) and (probability_1 < probability_0): + flip = True + + # Test for the second flip + if flip and (probability_1 > probability_0): + raise ValueError("Probability flipped twice") + + if flip: + assert probability_1 <= probability_0 + else: + assert probability_1 >= probability_0 + + +def test_cluster_murata_binned_mean(murata_binned_relation: MurataBinned): + for mass in np.linspace(7.0, 26.0, 2000): + for z in np.geomspace(1.0e-18, 2.0, 20): + test = murata_binned_relation.get_proxy_mean(mass, z) + + true = _observed_value( + (3.00, 0.086, 0.01), + mass, + z, + PIVOT_MASS * np.log(10.0), + np.log1p(PIVOT_Z), + ) + + assert test == pytest.approx(true, rel=1e-7, abs=0.0) + + +def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): + for mass in np.linspace(7.0, 26.0, 2000): + for z in np.geomspace(1.0e-18, 2.0, 20): + test = murata_binned_relation.get_proxy_sigma(mass, z) + + true = _observed_value( + (3.00, 0.07, 0.01), + mass, + z, + PIVOT_MASS * np.log(10.0), + np.log1p(PIVOT_Z), + ) + + assert test == pytest.approx(true, rel=1e-7, abs=0.0) From b2b07c3089216f189a1901a55a4aa71c9186bdfe Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 19 Oct 2023 09:47:43 -0700 Subject: [PATCH 24/80] Deferring the handling of building arguments, and how to read those arguments, to each integrator --- .../cluster_redshift_richness.py | 10 +- firecrown/integrator/__init__.py | 2 +- firecrown/integrator/integrator.py | 93 ++----------- firecrown/integrator/numcosmo_integrator.py | 131 ++++++++++++++++++ firecrown/integrator/scipy_integrator.py | 94 +++++++++++++ .../statistic/binned_cluster_number_counts.py | 88 +++--------- firecrown/models/cluster/abundance.py | 12 +- firecrown/models/cluster/kernel.py | 28 ++-- .../models/cluster/mass_proxy/gaussian.py | 8 +- firecrown/models/cluster/mass_proxy/murata.py | 1 - 10 files changed, 282 insertions(+), 185 deletions(-) create mode 100644 firecrown/integrator/numcosmo_integrator.py create mode 100644 firecrown/integrator/scipy_integrator.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index c6d1e6fda..788d4d898 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -5,7 +5,8 @@ import pyccl as ccl import sacc -from firecrown.integrator.integrator import NumCosmoIntegrator +from firecrown.integrator.numcosmo_integrator import NumCosmoIntegrator +from firecrown.integrator.scipy_integrator import ScipyIntegrator from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( BinnedClusterNumberCounts, @@ -19,7 +20,7 @@ Purity, SpectroscopicRedshift, ) -from firecrown.models.cluster.kernel.mass_proxy.murata import MurataBinned +from firecrown.models.cluster.mass_proxy.murata import MurataBinned def get_cluster_abundance(sky_area): @@ -35,8 +36,8 @@ def get_cluster_abundance(sky_area): mass_observable_kernel = MurataBinned(pivot_mass, pivot_redshift) cluster_abundance.add_kernel(mass_observable_kernel) - redshift_proxy_kernel = SpectroscopicRedshift() - # redshift_proxy_kernel = DESY1PhotometricRedshift() + # redshift_proxy_kernel = SpectroscopicRedshift() + redshift_proxy_kernel = DESY1PhotometricRedshift() cluster_abundance.add_kernel(redshift_proxy_kernel) # completeness_kernel = Completeness() @@ -53,6 +54,7 @@ def build_likelihood(build_parameters): Here we instantiate the number density (or mass function) object. """ integrator = NumCosmoIntegrator() + # integrator = ScipyIntegrator() # Pull params for the likelihood from build_parameters use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) diff --git a/firecrown/integrator/__init__.py b/firecrown/integrator/__init__.py index 7828e5789..e0fccee8a 100644 --- a/firecrown/integrator/__init__.py +++ b/firecrown/integrator/__init__.py @@ -1 +1 @@ -from .integrator import NumCosmoIntegrator, ScipyIntegrator, Integrator +from .integrator import Integrator diff --git a/firecrown/integrator/integrator.py b/firecrown/integrator/integrator.py index 75f6f77fe..be34360c4 100644 --- a/firecrown/integrator/integrator.py +++ b/firecrown/integrator/integrator.py @@ -1,89 +1,20 @@ -from numcosmo_py import Ncm -from typing import Tuple from abc import ABC, abstractmethod -from scipy.integrate import nquad -import numpy as np +from firecrown.models.cluster.kernel import ArgReader +from firecrown.models.cluster.abundance import ClusterAbundance class Integrator(ABC): - @abstractmethod - def integrate(self, integrand, bounds, bounds_map, extra_args): - pass - - -class ScipyIntegrator(Integrator): - def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): - super().__init__() - self._relative_tolerance = relative_tolerance - self._absolute_tolerance = absolute_tolerance - - def integrate(self, integrand, bounds, bounds_map, extra_args): - # TODO: Scipy passes the bounds unwrapped, while NumCosmo passes the bounds - # Wrapped. This means we need to adjust the called code to handle this. - # This isn't done yet. - cc = nquad( - integrand, - ranges=bounds, - args=(extra_args, bounds_map), - opts={ - "epsabs": self._absolute_tolerance, - "epsrel": self._relative_tolerance, - }, - )[0] - - return cc - - -class NumCosmoIntegrator(Integrator): - def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): - super().__init__() - self._relative_tolerance = relative_tolerance - self._absolute_tolerance = absolute_tolerance + arg_reader: ArgReader - def integrate(self, integrand, bounds, bounds_map, extra_args): + def __init__(self) -> None: super().__init__() - Ncm.cfg_init() - int_nd = CountsIntegralND( - len(bounds), - integrand, - extra_args, - bounds_map, - ) - int_nd.set_method(Ncm.IntegralNDMethod.P_V) - int_nd.set_reltol(self._relative_tolerance) - int_nd.set_abstol(self._absolute_tolerance) - res = Ncm.Vector.new(1) - err = Ncm.Vector.new(1) - bl, bu = zip(*bounds) - int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) - return res.get(0) - - -class CountsIntegralND(Ncm.IntegralND): - """Integral subclass used by the ClusterAbundance - to compute the integrals using numcosmo.""" - - def __init__(self, dim, fun, *args): - super().__init__() - self.dim = dim - self.fun = fun - self.args = args - - # pylint: disable-next=arguments-differ - def do_get_dimensions(self) -> Tuple[int, int]: - """Get number of dimensions.""" - return self.dim, 1 + @abstractmethod + def integrate( + self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits + ): + pass - # pylint: disable-next=arguments-differ - def do_integrand( - self, - x_vec: Ncm.Vector, - dim: int, - npoints: int, - _fdim: int, - fval_vec: Ncm.Vector, - ) -> None: - """Integrand function.""" - x = np.array(x_vec.dup_array()).reshape(npoints, dim) - fval_vec.set_array(self.fun(x, *self.args)) + @abstractmethod + def get_integration_bounds(self, integrand, bounds, extra_args): + pass diff --git a/firecrown/integrator/numcosmo_integrator.py b/firecrown/integrator/numcosmo_integrator.py new file mode 100644 index 000000000..2fa981949 --- /dev/null +++ b/firecrown/integrator/numcosmo_integrator.py @@ -0,0 +1,131 @@ +from numcosmo_py import Ncm +from typing import Tuple +import numpy as np +from firecrown.models.cluster.kernel import ArgReader, KernelType +from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.integrator import Integrator + + +class NumCosmoArgReader(ArgReader): + def __init__(self): + super().__init__() + self.integral_bounds = dict() + self.extra_args = dict() + + self.integral_bounds_idx = 0 + self.extra_args_idx = 1 + + def get_integral_bounds(self, int_args, kernel_type: KernelType): + bounds_values = int_args[self.integral_bounds_idx] + return bounds_values[:, self.integral_bounds[kernel_type.name]] + + def get_extra_args(self, int_args, kernel_type: KernelType): + extra_values = int_args[self.extra_args_idx] + return extra_values[self.extra_args[kernel_type.name]] + + +class NumCosmoIntegrator(Integrator): + def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): + super().__init__() + self._relative_tolerance = relative_tolerance + self._absolute_tolerance = absolute_tolerance + self.arg_reader = NumCosmoArgReader() + + def get_integration_bounds( + self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits + ): + self.arg_reader.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + integral_bounds = [ + (cl_abundance.min_mass, cl_abundance.max_mass), + (cl_abundance.min_z, cl_abundance.max_z), + ] + + # If any kernel is a dirac delta for z or M, just replace the + # true limits with the proxy limits + for kernel in cl_abundance.dirac_delta_kernels: + if kernel.kernel_type == KernelType.z_proxy: + integral_bounds[1] = z_proxy_limits + elif kernel.kernel_type == KernelType.mass_proxy: + integral_bounds[0] = mass_proxy_limits + + # If any kernel is not a dirac delta, integrate over the relevant limits + mapping_idx = len(self.arg_reader.integral_bounds.keys()) + for kernel in cl_abundance.integrable_kernels: + self.arg_reader.integral_bounds[kernel.kernel_type.name] = mapping_idx + mapping_idx += 1 + + if kernel.kernel_type == KernelType.z_proxy: + integral_bounds.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + integral_bounds.append(mass_proxy_limits) + + if kernel.integral_bounds is not None: + integral_bounds.append(kernel.integral_bounds) + + # Lastly, don't integrate any kernels with an analytic solution + # This means we pass in their limits as extra arguments to the integrator + extra_args = [] + self.arg_reader.extra_args = {} + for i, kernel in enumerate(cl_abundance.analytic_kernels): + self.arg_reader.extra_args[kernel.kernel_type.name] = i + + if kernel.kernel_type == KernelType.z_proxy: + extra_args.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + extra_args.append(mass_proxy_limits) + + if kernel.integral_bounds is not None: + extra_args.append(kernel.integral_bounds) + + return integral_bounds, extra_args + + def integrate(self, integrand, bounds, extra_args): + Ncm.cfg_init() + int_nd = CountsIntegralND( + len(bounds), + integrand, + extra_args, + self.arg_reader, + ) + int_nd.set_method(Ncm.IntegralNDMethod.P_V) + int_nd.set_reltol(self._relative_tolerance) + int_nd.set_abstol(self._absolute_tolerance) + res = Ncm.Vector.new(1) + err = Ncm.Vector.new(1) + + bl, bu = zip(*bounds) + int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) + return res.get(0) + + +class CountsIntegralND(Ncm.IntegralND): + """Integral subclass used by the ClusterAbundance + to compute the integrals using numcosmo.""" + + def __init__(self, dim, fun, *args): + super().__init__() + self.dim = dim + self.fun = fun + self.args = args + + # pylint: disable-next=arguments-differ + def do_get_dimensions(self) -> Tuple[int, int]: + """Get number of dimensions.""" + return self.dim, 1 + + # pylint: disable-next=arguments-differ + def do_integrand( + self, + x_vec: Ncm.Vector, + dim: int, + npoints: int, + _fdim: int, + fval_vec: Ncm.Vector, + ) -> None: + """Integrand function.""" + x = np.array(x_vec.dup_array()).reshape(npoints, dim) + fval_vec.set_array(self.fun(x, *self.args)) diff --git a/firecrown/integrator/scipy_integrator.py b/firecrown/integrator/scipy_integrator.py new file mode 100644 index 000000000..56fbe8ec4 --- /dev/null +++ b/firecrown/integrator/scipy_integrator.py @@ -0,0 +1,94 @@ +from scipy.integrate import nquad + +from firecrown.integrator import Integrator +from firecrown.models.cluster.kernel import ArgReader, KernelType +from firecrown.models.cluster.abundance import ClusterAbundance +import numpy as np + + +class ScipyIntegrator(Integrator): + def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): + super().__init__() + self._relative_tolerance = relative_tolerance + self._absolute_tolerance = absolute_tolerance + self.arg_reader = ScipyArgReader() + + def get_integration_bounds( + self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits + ): + self.arg_reader.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + integral_bounds = [ + (cl_abundance.min_mass, cl_abundance.max_mass), + (cl_abundance.min_z, cl_abundance.max_z), + ] + + # If any kernel is a dirac delta for z or M, just replace the + # true limits with the proxy limits + for kernel in cl_abundance.dirac_delta_kernels: + if kernel.kernel_type == KernelType.z_proxy: + integral_bounds[1] = z_proxy_limits + elif kernel.kernel_type == KernelType.mass_proxy: + integral_bounds[0] = mass_proxy_limits + + # If any kernel is not a dirac delta, integrate over the relevant limits + mapping_idx = len(self.arg_reader.integral_bounds.keys()) + for kernel in cl_abundance.integrable_kernels: + self.arg_reader.integral_bounds[kernel.kernel_type.name] = mapping_idx + mapping_idx += 1 + + if kernel.kernel_type == KernelType.z_proxy: + integral_bounds.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + integral_bounds.append(mass_proxy_limits) + + if kernel.integral_bounds is not None: + integral_bounds.append(kernel.integral_bounds) + + # Lastly, don't integrate any kernels with an analytic solution + # This means we pass in their limits as extra arguments to the integrator + extra_args = [] + self.arg_reader.extra_args = {} + + for kernel in cl_abundance.analytic_kernels: + self.arg_reader.extra_args[kernel.kernel_type.name] = mapping_idx + mapping_idx += 1 + if kernel.kernel_type == KernelType.z_proxy: + extra_args.append(z_proxy_limits) + elif kernel.kernel_type == KernelType.mass_proxy: + extra_args.append(mass_proxy_limits) + + if kernel.integral_bounds is not None: + extra_args.append(kernel.integral_bounds) + + return integral_bounds, extra_args + + def integrate(self, integrand, bounds, extra_args): + val = nquad( + integrand, + ranges=bounds, + args=(*extra_args, self.arg_reader), + opts={ + "epsabs": self._absolute_tolerance, + "epsrel": self._relative_tolerance, + }, + )[0] + + return val + + +class ScipyArgReader(ArgReader): + def __init__(self): + super().__init__() + self.integral_bounds = dict() + self.extra_args = dict() + self.integral_bounds_idx = 0 + + def get_integral_bounds(self, int_args, kernel_type: KernelType): + return np.array(int_args[self.integral_bounds[kernel_type.name]]) + + def get_extra_args(self, int_args, kernel_type: KernelType): + return int_args[self.extra_args[kernel_type.name]] diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 10ed56db1..f2b9b3395 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -3,12 +3,14 @@ import sacc from firecrown.integrator import Integrator -from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.abundance_data import AbundanceData -from firecrown.models.cluster.kernel import ArgsMapping, KernelType -from .statistic import Statistic, DataVector, TheoryVector -from .source.source import SourceSystematic -from ....modeling_tools import ModelingTools +from firecrown.likelihood.gauss_family.statistic.statistic import ( + Statistic, + DataVector, + TheoryVector, +) +from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic +from firecrown.modeling_tools import ModelingTools import numpy as np import cProfile @@ -75,87 +77,29 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_masses = [] if self.use_cluster_counts or self.use_mean_log_mass: - # self.pr.enable() for z_proxy_limits, mass_proxy_limits in self.bin_limits: - bounds, extra_args, args_mapping = self.get_integration_bounds( - tools.cluster_abundance, z_proxy_limits, mass_proxy_limits - ) - integrand = tools.cluster_abundance.get_integrand() - counts = self.integrator.integrate( - integrand, bounds, args_mapping, extra_args + bounds, extra_args = self.integrator.get_integration_bounds( + tools.cluster_abundance, z_proxy_limits, mass_proxy_limits ) + counts = self.integrator.integrate(integrand, bounds, extra_args) cluster_counts.append(counts) - # self.pr.disable() - # self.pr.dump_stats("profile.prof") theory_vector_list += cluster_counts if self.use_mean_log_mass: for (z_proxy_limits, mass_proxy_limits), counts in zip( self.bin_limits, cluster_counts ): - bounds, extra_args, args_mapping = self.get_integration_bounds( - z_proxy_limits, mass_proxy_limits - ) - - integrand = tools.cluster_abundance.get_integrand() - unnormalized_mass = self.integrator.integrate( - integrand, bounds, args_mapping, extra_args + integrand = tools.cluster_abundance.get_integrand(avg_mass=True) + bounds, extra_args = self.integrator.get_integration_bounds( + tools.cluster_abundance, z_proxy_limits, mass_proxy_limits ) - cluster_mass = unnormalized_mass / counts - cluster_masses.append(cluster_mass) + total_mass = self.integrator.integrate(integrand, bounds, extra_args) + mean_mass = total_mass / counts + cluster_masses.append(mean_mass) theory_vector_list += cluster_masses return TheoryVector.from_list(theory_vector_list) - - def get_integration_bounds( - self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits - ): - args_mapping = ArgsMapping() - args_mapping.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - - integral_bounds = [ - (cl_abundance.min_mass, cl_abundance.max_mass), - (cl_abundance.min_z, cl_abundance.max_z), - ] - - # If any kernel is a dirac delta for z or M, just replace the - # true limits with the proxy limits - for kernel in cl_abundance.dirac_delta_kernels: - if kernel.kernel_type == KernelType.z_proxy: - integral_bounds[1] = z_proxy_limits - elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds[0] = mass_proxy_limits - - # If any kernel is not a dirac delta, integrate over the relevant limits - mapping_idx = len(args_mapping.integral_bounds.keys()) - for kernel in cl_abundance.integrable_kernels: - args_mapping.integral_bounds[kernel.kernel_type.name] = mapping_idx - mapping_idx += 1 - - if kernel.kernel_type == KernelType.z_proxy: - integral_bounds.append(z_proxy_limits) - elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds.append(mass_proxy_limits) - - if kernel.integral_bounds is not None: - integral_bounds.append(kernel.integral_bounds) - - # Lastly, don't integrate any kernels with an analytic solution - # This means we pass in their limits as extra arguments to the integrator - extra_args = [] - for i, kernel in enumerate(cl_abundance.analytic_kernels): - args_mapping.extra_args[kernel.kernel_type.name] = i - - if kernel.kernel_type == KernelType.z_proxy: - extra_args.append(z_proxy_limits) - elif kernel.kernel_type == KernelType.mass_proxy: - extra_args.append(mass_proxy_limits) - - if kernel.integral_bounds is not None: - extra_args.append(kernel.integral_bounds) - - return integral_bounds, extra_args, args_mapping diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 47e198906..833b4fa23 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -2,13 +2,11 @@ from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader import numpy as np from firecrown.parameters import ParamsMap import numpy.typing as npt -# TODO: Consider emulators for mass function - class ClusterAbundance(object): @property @@ -91,7 +89,6 @@ def mass_function( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] ) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) - return_vals = [] for m, a in zip(mass, scale_factor): val = self._hmf_cache.get((m, a)) @@ -100,15 +97,20 @@ def mass_function( self._hmf_cache[(m, a)] = val return_vals.append(val) + if len(return_vals) == 1: + return return_vals[0] return return_vals def get_integrand(self, avg_mass=False, avg_redshift=False): def integrand(*int_args): - args_map: ArgsMapping = int_args[-1] + args_map: ArgReader = int_args[-1] z = args_map.get_integral_bounds(int_args, KernelType.z) mass = args_map.get_integral_bounds(int_args, KernelType.mass) + z = np.atleast_1d(z) + mass = np.atleast_1d(mass) + integrand = self.comoving_volume(z) * self.mass_function(mass, z) if avg_mass: integrand *= mass diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 88b32b3fa..4c49a5e95 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,10 +1,9 @@ -from abc import ABC +from abc import ABC, abstractmethod from enum import Enum from typing import List, Tuple import numpy as np from firecrown.updatable import Updatable -import pdb class KernelType(Enum): @@ -16,21 +15,18 @@ class KernelType(Enum): purity = 6 -class ArgsMapping: +class ArgReader(ABC): def __init__(self): self.integral_bounds = dict() self.extra_args = dict() - self.integral_bounds_idx = 0 - self.extra_args_idx = 1 - + @abstractmethod def get_integral_bounds(self, int_args, kernel_type: KernelType): - bounds_values = int_args[self.integral_bounds_idx] - return bounds_values[:, self.integral_bounds[kernel_type.name]] + pass + @abstractmethod def get_extra_args(self, int_args, kernel_type: KernelType): - extra_values = int_args[self.extra_args_idx] - return extra_values[self.extra_args[kernel_type.name]] + pass class Kernel(Updatable, ABC): @@ -47,7 +43,7 @@ def __init__( self.kernel_type = kernel_type self.has_analytic_sln = has_analytic_sln - def distribution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgReader): raise NotImplementedError() @@ -55,7 +51,7 @@ class Completeness(Kernel): def __init__(self): super().__init__(KernelType.completeness) - def distribution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgReader): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) @@ -73,7 +69,7 @@ class Purity(Kernel): def __init__(self): super().__init__(KernelType.purity) - def distribution(self, args: List[float], args_index_map: ArgsMapping): + def distribution(self, args: List[float], args_index_map: ArgReader): mass_proxy = args_index_map.get_integral_bounds(args, KernelType.mass_proxy) z = args_index_map.get_integral_bounds(args, KernelType.z) @@ -96,7 +92,7 @@ class TrueMass(Kernel): def __init__(self): super().__init__(KernelType.mass_proxy, True) - def distribution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgReader): return 1.0 @@ -104,7 +100,7 @@ class SpectroscopicRedshift(Kernel): def __init__(self): super().__init__(KernelType.z_proxy, True) - def distribution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgReader): return 1.0 @@ -113,7 +109,7 @@ def __init__(self): super().__init__(KernelType.z_proxy) self.sigma_0 = 0.05 - def distribution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgReader): z = args_map.get_integral_bounds(args, KernelType.z) z_proxy = args_map.get_integral_bounds(args, KernelType.z_proxy) diff --git a/firecrown/models/cluster/mass_proxy/gaussian.py b/firecrown/models/cluster/mass_proxy/gaussian.py index 02ce162c9..3c8c46dc2 100644 --- a/firecrown/models/cluster/mass_proxy/gaussian.py +++ b/firecrown/models/cluster/mass_proxy/gaussian.py @@ -1,9 +1,7 @@ from typing import List - import numpy as np from scipy import special - -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping +from firecrown.models.cluster.kernel import ArgReader, Kernel, KernelType class MassRichnessGaussian(Kernel): @@ -15,7 +13,7 @@ def get_proxy_sigma(self, mass, z): """Return observed scatter corrected by redshift and mass.""" return NotImplementedError - def _distribution_binned(self, args: List[float], args_map: ArgsMapping): + def _distribution_binned(self, args: List[float], args_map: ArgReader): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) mass_proxy_limits = args_map.get_extra_args(args, self.kernel_type) @@ -43,7 +41,7 @@ def _distribution_binned(self, args: List[float], args_map: ArgsMapping): return return_vals - def _distribution_unbinned(self, args: List[float], args_map: ArgsMapping): + def _distribution_unbinned(self, args: List[float], args_map: ArgReader): mass = args_map.get_integral_bounds(args, KernelType.mass) z = args_map.get_integral_bounds(args, KernelType.z) mass_proxy = args_map.get_extra_args(args, self.kernel_type) diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py index ca5a05552..d656f21eb 100644 --- a/firecrown/models/cluster/mass_proxy/murata.py +++ b/firecrown/models/cluster/mass_proxy/murata.py @@ -63,7 +63,6 @@ def get_proxy_sigma(self, mass, z): ) -### used to be MassRichnessMuSigma ### class MurataBinned(MurataCore): def distribution(self, args, args_map): return self._distribution_binned(args, args_map) From fe424952ad193b26175cafc6a94eed607836122b Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 19 Oct 2023 09:59:48 -0700 Subject: [PATCH 25/80] Fixing unit tests for new args reader changes --- tests/test_cluster_abundance.py | 4 ++-- tests/test_cluster_kernels.py | 33 +++++++++++++++++++++++------ tests/test_cluster_mass_richness.py | 19 +++++++++++++++-- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 31300d382..1406068b7 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,6 +1,6 @@ import pytest from firecrown.models.cluster.abundance import ClusterAbundance -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgsMapping +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader import pyccl import numpy as np from typing import List, Tuple @@ -24,7 +24,7 @@ def __init__( ): super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) - def distribution(self, args: List[float], args_map: ArgsMapping): + def distribution(self, args: List[float], args_map: ArgReader): return 1.0 diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index a5840ad58..74ff9a1e1 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -8,10 +8,25 @@ DESY1PhotometricRedshift, SpectroscopicRedshift, TrueMass, - ArgsMapping, + ArgReader, ) +class MockArgsReader(ArgReader): + def __init__(self): + super().__init__() + self.integral_bounds_idx = 0 + self.extra_args_idx = 1 + + def get_integral_bounds(self, int_args, kernel_type: KernelType): + bounds_values = int_args[self.integral_bounds_idx] + return bounds_values[:, self.integral_bounds[kernel_type.name]] + + def get_extra_args(self, int_args, kernel_type: KernelType): + extra_values = int_args[self.extra_args_idx] + return extra_values[self.extra_args[kernel_type.name]] + + def test_create_desy1_photometric_redshift_kernel(): drk = DESY1PhotometricRedshift() assert isinstance(drk, Kernel) @@ -59,12 +74,18 @@ def test_create_purity_kernel(): def test_spec_z_distribution(): srk = SpectroscopicRedshift() - assert srk.distribution([0.5], ArgsMapping()) == 1.0 + assert srk.distribution([0.5], MockArgsReader()) == 1.0 def test_true_mass_distribution(): tmk = TrueMass() - assert tmk.distribution([0.5], ArgsMapping()) == 1.0 + assert tmk.distribution([0.5], MockArgsReader()) == 1.0 + + +def test_create_arg_reader(): + mr = MockArgsReader() + assert mr.integral_bounds == dict() + assert mr.extra_args == dict() def test_purity_distribution(): @@ -73,7 +94,7 @@ def test_purity_distribution(): z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) arguments = np.array(list(zip(mass_proxy, z))) - map = ArgsMapping() + map = MockArgsReader() map.integral_bounds = {KernelType.mass_proxy.name: 0, KernelType.z.name: 1} truth = np.array( @@ -102,7 +123,7 @@ def test_completeness_distribution(): z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) bounds = np.array(list(zip(mass, z))) - map = ArgsMapping() + map = MockArgsReader() map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} truth = np.array( @@ -134,7 +155,7 @@ def test_des_photoz_kernel_distribution(): bounds = np.array(list(zip(mass, z, z_proxy))) - map = ArgsMapping() + map = MockArgsReader() map.integral_bounds = { KernelType.mass.name: 0, KernelType.z.name: 1, diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index e5b064bb3..ff794db87 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -4,13 +4,28 @@ from firecrown.models.cluster.kernel import ( KernelType, Kernel, - ArgsMapping, + ArgReader, ) PIVOT_Z = 0.6 PIVOT_MASS = 14.625862906 +class MockArgsReader(ArgReader): + def __init__(self): + super().__init__() + self.integral_bounds_idx = 0 + self.extra_args_idx = 1 + + def get_integral_bounds(self, int_args, kernel_type: KernelType): + bounds_values = int_args[self.integral_bounds_idx] + return bounds_values[:, self.integral_bounds[kernel_type.name]] + + def get_extra_args(self, int_args, kernel_type: KernelType): + extra_values = int_args[self.extra_args_idx] + return extra_values[self.extra_args[kernel_type.name]] + + @pytest.fixture(name="murata_binned_relation") def fixture_cluster_mass_rich() -> MurataBinned: """Initialize cluster object.""" @@ -71,7 +86,7 @@ def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned args1 = [np.array([[logM_0, z]]), extra_args] args2 = [np.array([[logM_1, z]]), extra_args] - args_map = ArgsMapping() + args_map = MockArgsReader() args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} args_map.extra_args = {KernelType.mass_proxy.name: 0} From 5fd98e6b13bcae30a005bc39f2279c22e6767444 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 19 Oct 2023 11:17:43 -0700 Subject: [PATCH 26/80] Added covering unit tests for the integrator classes --- firecrown/integrator/integrator.py | 12 +- firecrown/models/cluster/kernel.py | 6 +- .../models/cluster/mass_proxy/gaussian.py | 2 - tests/test_cluster_args_mapping.py | 56 -- tests/test_cluster_mass_richness.py | 1 + tests/test_integrator.py | 633 ++++++++++++++++++ 6 files changed, 643 insertions(+), 67 deletions(-) delete mode 100644 tests/test_cluster_args_mapping.py create mode 100644 tests/test_integrator.py diff --git a/firecrown/integrator/integrator.py b/firecrown/integrator/integrator.py index be34360c4..557639e80 100644 --- a/firecrown/integrator/integrator.py +++ b/firecrown/integrator/integrator.py @@ -10,11 +10,11 @@ def __init__(self) -> None: super().__init__() @abstractmethod - def integrate( - self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits - ): - pass + def integrate(self, integrand, bounds, extra_args): + """Integrate the integrand over the bounds and include extra_args to integral""" @abstractmethod - def get_integration_bounds(self, integrand, bounds, extra_args): - pass + def get_integration_bounds( + self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits + ): + """Extract the limits of integration and extra arguments for the integral""" diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 4c49a5e95..a8f128b95 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -22,11 +22,11 @@ def __init__(self): @abstractmethod def get_integral_bounds(self, int_args, kernel_type: KernelType): - pass + """Returns the current differential value for KernelType""" @abstractmethod def get_extra_args(self, int_args, kernel_type: KernelType): - pass + """Returns the extra arguments passed into the integral for KernelType""" class Kernel(Updatable, ABC): @@ -44,7 +44,7 @@ def __init__( self.has_analytic_sln = has_analytic_sln def distribution(self, args: List[float], args_map: ArgReader): - raise NotImplementedError() + """The functional form of the distribution or spread of this kernel""" class Completeness(Kernel): diff --git a/firecrown/models/cluster/mass_proxy/gaussian.py b/firecrown/models/cluster/mass_proxy/gaussian.py index 3c8c46dc2..4ea01938a 100644 --- a/firecrown/models/cluster/mass_proxy/gaussian.py +++ b/firecrown/models/cluster/mass_proxy/gaussian.py @@ -7,11 +7,9 @@ class MassRichnessGaussian(Kernel): def get_proxy_mean(self, mass, z): """Return observed quantity corrected by redshift and mass.""" - return NotImplementedError def get_proxy_sigma(self, mass, z): """Return observed scatter corrected by redshift and mass.""" - return NotImplementedError def _distribution_binned(self, args: List[float], args_map: ArgReader): mass = args_map.get_integral_bounds(args, KernelType.mass) diff --git a/tests/test_cluster_args_mapping.py b/tests/test_cluster_args_mapping.py deleted file mode 100644 index 8a26b2e70..000000000 --- a/tests/test_cluster_args_mapping.py +++ /dev/null @@ -1,56 +0,0 @@ -import numpy as np -from firecrown.models.cluster.kernel import ( - KernelType, - ArgsMapping, -) - - -def test_args_mapper_extra_args(): - args_map = ArgsMapping() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - - extra_args = [(1, 2, 3), ("hello world")] - args_map.extra_args = {KernelType.mass_proxy.name: 0, KernelType.z_proxy.name: 1} - - integral_bounds = np.array(list(zip(mass, z))) - int_args = [integral_bounds, extra_args] - - assert args_map.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) - assert args_map.get_extra_args(int_args, KernelType.z_proxy) == "hello world" - - -def test_args_mapper_integral_bounds(): - args_map = ArgsMapping() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) - mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) - - args_map.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - } - - integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) - int_args = [integral_bounds] - - assert (mass == args_map.get_integral_bounds(int_args, KernelType.mass)).all() - assert (z == args_map.get_integral_bounds(int_args, KernelType.z)).all() - assert (z_proxy == args_map.get_integral_bounds(int_args, KernelType.z_proxy)).all() - assert ( - mass_proxy == args_map.get_integral_bounds(int_args, KernelType.mass_proxy) - ).all() - - -def test_create_args_mapping(): - am = ArgsMapping() - assert am.integral_bounds == dict() - assert am.extra_args == dict() - assert am.integral_bounds_idx == 0 - assert am.extra_args_idx == 1 diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index ff794db87..52b480eb2 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,5 +1,6 @@ import pytest import numpy as np +from firecrown.models.cluster.mass_proxy.gaussian import MassRichnessGaussian from firecrown.models.cluster.mass_proxy.murata import MurataBinned, _observed_value from firecrown.models.cluster.kernel import ( KernelType, diff --git a/tests/test_integrator.py b/tests/test_integrator.py new file mode 100644 index 000000000..3d2cfdd5e --- /dev/null +++ b/tests/test_integrator.py @@ -0,0 +1,633 @@ +import numpy as np +import pytest +from typing import List, Tuple +from firecrown.integrator.numcosmo_integrator import ( + NumCosmoArgReader, + NumCosmoIntegrator, +) +from firecrown.integrator.scipy_integrator import ScipyArgReader, ScipyIntegrator +from firecrown.models.cluster.kernel import KernelType, Kernel, ArgReader +from firecrown.models.cluster.abundance import ClusterAbundance + + +@pytest.fixture(name="cl_abundance") +def fixture_cl_abundance(): + cl_abundance = ClusterAbundance( + min_z=0, + max_z=2, + min_mass=13, + max_mass=17, + sky_area=100, + halo_mass_function=None, + ) + return cl_abundance + + +class MockKernel(Kernel): + def __init__( + self, + kernel_type: KernelType, + is_dirac_delta: bool = False, + has_analytic_sln: bool = False, + integral_bounds: List[Tuple[float, float]] = None, + ): + super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) + + def distribution(self, args: List[float], arg_reader: ArgReader): + return 1.0 + + +def test_numcosmo_argreader_extra_args(): + arg_reader = NumCosmoArgReader() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + arg_reader.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + extra_args = [(1, 2, 3), ("hello world")] + arg_reader.extra_args = {KernelType.mass_proxy.name: 0, KernelType.z_proxy.name: 1} + + integral_bounds = np.array(list(zip(mass, z))) + int_args = [integral_bounds, extra_args] + + assert arg_reader.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) + assert arg_reader.get_extra_args(int_args, KernelType.z_proxy) == "hello world" + + +def test_numcosmo_argreader_integral_bounds(): + arg_reader = NumCosmoArgReader() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) + mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) + + arg_reader.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + } + + integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) + int_args = [integral_bounds] + + assert (mass == arg_reader.get_integral_bounds(int_args, KernelType.mass)).all() + assert (z == arg_reader.get_integral_bounds(int_args, KernelType.z)).all() + assert ( + z_proxy == arg_reader.get_integral_bounds(int_args, KernelType.z_proxy) + ).all() + assert ( + mass_proxy == arg_reader.get_integral_bounds(int_args, KernelType.mass_proxy) + ).all() + + +def test_create_numcosmo_argreader(): + am = NumCosmoArgReader() + assert am.integral_bounds == dict() + assert am.extra_args == dict() + assert am.integral_bounds_idx == 0 + assert am.extra_args_idx == 1 + + +def test_scipy_argreader_extra_args(): + arg_reader = ScipyArgReader() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + arg_reader.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + extra_args = [(1, 2, 3), ("hello world")] + arg_reader.extra_args = {KernelType.mass_proxy.name: 2, KernelType.z_proxy.name: 3} + + for m_i, z_i in list(zip(mass, z)): + int_args = [m_i, z_i, *extra_args] + + assert arg_reader.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) + assert arg_reader.get_extra_args(int_args, KernelType.z_proxy) == "hello world" + + +def test_scipy_argreader_integral_bounds(): + arg_reader = ScipyArgReader() + + mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) + mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) + + arg_reader.integral_bounds = { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + } + + for m_i, z_i, zp_i, mp_i in list(zip(mass, z, z_proxy, mass_proxy)): + int_args = [m_i, z_i, zp_i, mp_i] + assert m_i == arg_reader.get_integral_bounds(int_args, KernelType.mass) + assert z_i == arg_reader.get_integral_bounds(int_args, KernelType.z) + assert zp_i == arg_reader.get_integral_bounds(int_args, KernelType.z_proxy) + assert mp_i == arg_reader.get_integral_bounds(int_args, KernelType.mass_proxy) + + +def test_create_scipy_argreader(): + am = ScipyArgReader() + assert am.integral_bounds == dict() + assert am.extra_args == dict() + assert am.integral_bounds_idx == 0 + + +def test_numcosmo_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): + nci = NumCosmoIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + +def test_numcosmo_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): + nci = NumCosmoIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + dd_kernel = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [mass_limits, (0, 2)] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + cl_abundance.kernels.clear() + dd_kernel = MockKernel(KernelType.z_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [(13, 17), z_limits] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [mass_limits, z_limits] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + +def test_numcosmo_get_integration_bounds_integrable_kernels( + cl_abundance: ClusterAbundance, +): + nci = NumCosmoIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + ig_kernel = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 3 + assert bounds == [(13, 17), (0, 2), mass_limits] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.mass_proxy.name: 2, + } + + cl_abundance.kernels.clear() + ig_kernel = MockKernel(KernelType.z_proxy) + cl_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 3 + assert bounds == [(13, 17), (0, 2), z_limits] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + } + + ig_kernel2 = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(ig_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 4 + assert bounds == [(13, 17), (0, 2), z_limits, mass_limits] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + } + + p_kernel = MockKernel(KernelType.purity, integral_bounds=(0, 1)) + cl_abundance.add_kernel(p_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 5 + assert bounds == [(13, 17), (0, 2), z_limits, mass_limits, (0, 1)] + assert len(nci.arg_reader.extra_args) == 0 + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + KernelType.purity.name: 4, + } + + +def test_numcosmo_get_integration_bounds_analytic_slns( + cl_abundance: ClusterAbundance, +): + nci = NumCosmoIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + a_kernel = MockKernel(KernelType.mass_proxy, has_analytic_sln=True) + cl_abundance.add_kernel(a_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 1 + assert extra_args == [mass_limits] + assert nci.arg_reader.extra_args == {KernelType.mass_proxy.name: 0} + + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) + cl_abundance.add_kernel(a_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 2 + assert extra_args == [mass_limits, z_limits] + assert nci.arg_reader.extra_args == { + KernelType.mass_proxy.name: 0, + KernelType.z_proxy.name: 1, + } + + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + a_kernel3 = MockKernel( + KernelType.purity, has_analytic_sln=True, integral_bounds=(0, 1) + ) + cl_abundance.add_kernel(a_kernel3) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = nci.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 3 + assert extra_args == [mass_limits, z_limits, (0, 1)] + assert nci.arg_reader.extra_args == { + KernelType.mass_proxy.name: 0, + KernelType.z_proxy.name: 1, + KernelType.purity.name: 2, + } + + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert nci.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + +def test_scipy_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + +def test_scipy_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + dd_kernel = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [mass_limits, (0, 2)] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + cl_abundance.kernels.clear() + dd_kernel = MockKernel(KernelType.z_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [(13, 17), z_limits] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 2 + assert bounds == [mass_limits, z_limits] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + +def test_scipy_get_integration_bounds_integrable_kernels( + cl_abundance: ClusterAbundance, +): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + ig_kernel = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 3 + assert bounds == [(13, 17), (0, 2), mass_limits] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.mass_proxy.name: 2, + } + + cl_abundance.kernels.clear() + ig_kernel = MockKernel(KernelType.z_proxy) + cl_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 3 + assert bounds == [(13, 17), (0, 2), z_limits] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + } + + ig_kernel2 = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(ig_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 4 + assert bounds == [(13, 17), (0, 2), z_limits, mass_limits] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + } + + p_kernel = MockKernel(KernelType.purity, integral_bounds=(0, 1)) + cl_abundance.add_kernel(p_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 0 + assert len(bounds) == 5 + assert bounds == [(13, 17), (0, 2), z_limits, mass_limits, (0, 1)] + assert len(spi.arg_reader.extra_args) == 0 + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + KernelType.z_proxy.name: 2, + KernelType.mass_proxy.name: 3, + KernelType.purity.name: 4, + } + + +def test_scipy_get_integration_bounds_analytic_slns( + cl_abundance: ClusterAbundance, +): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + a_kernel = MockKernel(KernelType.mass_proxy, has_analytic_sln=True) + cl_abundance.add_kernel(a_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 1 + assert extra_args == [mass_limits] + assert spi.arg_reader.extra_args == {KernelType.mass_proxy.name: 2} + + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) + cl_abundance.add_kernel(a_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 2 + assert extra_args == [mass_limits, z_limits] + assert spi.arg_reader.extra_args == { + KernelType.mass_proxy.name: 2, + KernelType.z_proxy.name: 3, + } + + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } + + a_kernel3 = MockKernel( + KernelType.purity, has_analytic_sln=True, integral_bounds=(0, 1) + ) + cl_abundance.add_kernel(a_kernel3) + + for z_limits, mass_limits in zip(z_bins, m_bins): + bounds, extra_args = spi.get_integration_bounds( + cl_abundance, z_limits, mass_limits + ) + + assert len(extra_args) == 3 + assert extra_args == [mass_limits, z_limits, (0, 1)] + assert spi.arg_reader.extra_args == { + KernelType.mass_proxy.name: 2, + KernelType.z_proxy.name: 3, + KernelType.purity.name: 4, + } + + assert len(bounds) == 2 + assert bounds == [(13, 17), (0, 2)] + assert spi.arg_reader.integral_bounds == { + KernelType.mass.name: 0, + KernelType.z.name: 1, + } From 0ee47eb4ff9cf3638f938840af9ccdb89bd5053d Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 19 Oct 2023 13:45:09 -0700 Subject: [PATCH 27/80] Added test for unbinned Murata --- tests/test_cluster_mass_richness.py | 58 +++++++++- tests/test_clustermodel.py | 168 ---------------------------- 2 files changed, 57 insertions(+), 169 deletions(-) delete mode 100644 tests/test_clustermodel.py diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 52b480eb2..766053254 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,7 +1,11 @@ import pytest import numpy as np from firecrown.models.cluster.mass_proxy.gaussian import MassRichnessGaussian -from firecrown.models.cluster.mass_proxy.murata import MurataBinned, _observed_value +from firecrown.models.cluster.mass_proxy.murata import ( + MurataBinned, + MurataUnbinned, + _observed_value, +) from firecrown.models.cluster.kernel import ( KernelType, Kernel, @@ -45,6 +49,24 @@ def fixture_cluster_mass_rich() -> MurataBinned: return mr +@pytest.fixture(name="murata_unbinned_relation") +def fixture_cluster_mass_rich() -> MurataUnbinned: + """Initialize cluster object.""" + + mr = MurataUnbinned(PIVOT_MASS, PIVOT_Z) + + # Set the parameters to the values used in the test + # they should be such that the variance is always positive. + mr.mu_p0 = 3.00 + mr.mu_p1 = 0.086 + mr.mu_p2 = 0.01 + mr.sigma_p0 = 3.0 + mr.sigma_p1 = 0.07 + mr.sigma_p2 = 0.01 + + return mr + + def test_create_musigma_kernel(): mb = MurataBinned(1, 1) assert isinstance(mb, Kernel) @@ -144,3 +166,37 @@ def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): ) assert test == pytest.approx(true, rel=1e-7, abs=0.0) + + +def test_cluster_murata_unbinned_distribution(murata_unbinned_relation: MurataUnbinned): + logM_array = np.linspace(7.0, 26.0, 2000) + for z in np.geomspace(1.0e-18, 2.0, 20): + flip = False + for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): + extra_args = [2.5] + + args1 = [np.array([[logM_0, z]]), extra_args] + args2 = [np.array([[logM_1, z]]), extra_args] + + args_map = MockArgsReader() + args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + args_map.extra_args = {KernelType.mass_proxy.name: 0} + + probability_0 = murata_unbinned_relation.distribution(args1, args_map) + probability_1 = murata_unbinned_relation.distribution(args2, args_map) + + # Probability density should be initially monotonically increasing + # and then monotonically decreasing. It should flip only once. + + # Test for the flip + if (not flip) and (probability_1 < probability_0): + flip = True + + # Test for the second flip + if flip and (probability_1 > probability_0): + raise ValueError("Probability flipped twice") + + if flip: + assert probability_1 <= probability_0 + else: + assert probability_1 >= probability_0 diff --git a/tests/test_clustermodel.py b/tests/test_clustermodel.py deleted file mode 100644 index 99cc492ad..000000000 --- a/tests/test_clustermodel.py +++ /dev/null @@ -1,168 +0,0 @@ -# """Test cluster mass function computations.""" - -# import itertools -# import numpy as np -# import pytest - -# from firecrown.models.cluster_mass_rich_proxy import ( -# ClusterMassRich, -# ClusterMassRichBinArgument, -# ClusterMassRichPointArgument, -# ) - - -# @pytest.fixture(name="cluster_mass_rich") -# def fixture_cluster_mass_rich() -> ClusterMassRich: -# """Initialize cluster object.""" -# pivot_redshift = 0.6 -# pivot_mass = 14.625862906 - -# cluster_mass_rich = ClusterMassRich(pivot_mass, pivot_redshift) - -# # Set the parameters to the values used in the test -# # they should be such that the variance is always positive. -# cluster_mass_rich.mu_p0 = 3.00 -# cluster_mass_rich.mu_p1 = 0.086 -# cluster_mass_rich.mu_p2 = 0.01 -# cluster_mass_rich.sigma_p0 = 3.0 -# cluster_mass_rich.sigma_p1 = 0.07 -# cluster_mass_rich.sigma_p2 = 0.01 - -# return cluster_mass_rich - - -# def test_cluster_mass_parameters_function_z(): -# for z in np.geomspace(1.0e-18, 2.0, 20): -# f_z = ClusterMassRich.cluster_mass_parameters_function( -# 0.0, 0.0, (0.0, 0.0, 1.0), 0.0, z -# ) -# assert f_z == pytest.approx(np.log1p(z), 1.0e-7, 0.0) - - -# def test_cluster_mass_parameters_function_M(): -# for logM in np.linspace(10.0, 16.0, 20): -# f_logM = ClusterMassRich.cluster_mass_parameters_function( -# 0.0, 0.0, (0.0, 1.0, 0.0), logM, 0.0 -# ) - -# assert f_logM == pytest.approx(logM * np.log(10.0), 1.0e-7, 0.0) - - -# def test_cluster_mass_lnM_obs_mu_sigma(cluster_mass_rich: ClusterMassRich): -# for logM, z in itertools.product( -# np.linspace(10.0, 16.0, 20), np.geomspace(1.0e-18, 2.0, 20) -# ): -# lnM_obs_mu_direct = ClusterMassRich.cluster_mass_parameters_function( -# cluster_mass_rich.log_pivot_mass, -# cluster_mass_rich.log1p_pivot_redshift, -# (cluster_mass_rich.mu_p0, cluster_mass_rich.mu_p1, cluster_mass_rich.mu_p2), -# logM, -# z, -# ) -# sigma_direct = ClusterMassRich.cluster_mass_parameters_function( -# cluster_mass_rich.log_pivot_mass, -# cluster_mass_rich.log1p_pivot_redshift, -# ( -# cluster_mass_rich.sigma_p0, -# cluster_mass_rich.sigma_p1, -# cluster_mass_rich.sigma_p2, -# ), -# logM, -# z, -# ) -# lnM_obs_mu, sigma = cluster_mass_rich.cluster_mass_lnM_obs_mu_sigma(logM, z) - -# assert lnM_obs_mu == pytest.approx(lnM_obs_mu_direct, 1.0e-7, 0.0) -# assert sigma == pytest.approx(sigma_direct, 1.0e-7, 0.0) - - -# def test_cluster_richness_bins_invalid_edges(cluster_mass_rich: ClusterMassRich): -# rich_bin_edges = np.array([20]) - -# with pytest.raises(ValueError): -# _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - -# rich_bin_edges = np.array([20, 30, 10]) -# with pytest.raises(ValueError): -# _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - -# rich_bin_edges = np.array([20, 30, 30]) -# with pytest.raises(ValueError): -# _ = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - - -# def test_cluster_richness_bins(cluster_mass_rich: ClusterMassRich): -# rich_bin_edges = np.array([20, 40, 60, 80]) - -# richness_bins = cluster_mass_rich.gen_bins_by_array(rich_bin_edges) - -# for Rl, Ru, richness_bin in zip( -# rich_bin_edges[:-1], rich_bin_edges[1:], richness_bins -# ): -# assert isinstance(richness_bin, ClusterMassRichBinArgument) -# assert Rl == richness_bin.logM_obs_lower -# assert Ru == richness_bin.logM_obs_upper - - -# def test_cluster_bin_probability(cluster_mass_rich: ClusterMassRich): -# cluser_mass_rich_bin_argument = ClusterMassRichBinArgument( -# cluster_mass_rich, 13.0, 17.0, 1.0, 5.0 -# ) - -# logM_array = np.linspace(7.0, 26.0, 2000) -# for z in np.geomspace(1.0e-18, 2.0, 20): -# flip = False -# for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): -# probability_0 = cluser_mass_rich_bin_argument.p(logM_0, z) -# probability_1 = cluser_mass_rich_bin_argument.p(logM_1, z) - -# assert probability_0 >= 0 -# assert probability_1 >= 0 - -# # Probability should be initially monotonically increasing -# # and then monotonically decreasing. It should flip only once. - -# # Test for the flip -# if (not flip) and (probability_1 < probability_0): -# flip = True - -# # Test for the second flip -# if flip and (probability_1 > probability_0): -# raise ValueError("Probability flipped twice") - -# if flip: -# assert probability_1 <= probability_0 -# else: -# assert probability_1 >= probability_0 - - -# def test_cluster_point_probability(cluster_mass_rich: ClusterMassRich): -# cluser_mass_rich_bin_argument = ClusterMassRichPointArgument( -# cluster_mass_rich, 13.0, 17.0, 2.5 -# ) - -# logM_array = np.linspace(7.0, 26.0, 2000) -# for z in np.geomspace(1.0e-18, 2.0, 20): -# flip = False -# for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): -# probability_0 = cluser_mass_rich_bin_argument.p(logM_0, z) -# probability_1 = cluser_mass_rich_bin_argument.p(logM_1, z) - -# assert probability_0 >= 0 -# assert probability_1 >= 0 - -# # Probability density should be initially monotonically increasing -# # and then monotonically decreasing. It should flip only once. - -# # Test for the flip -# if (not flip) and (probability_1 < probability_0): -# flip = True - -# # Test for the second flip -# if flip and (probability_1 > probability_0): -# raise ValueError("Probability flipped twice") - -# if flip: -# assert probability_1 <= probability_0 -# else: -# assert probability_1 >= probability_0 From e820504ff6e70cc51c8d289ab0b4ba43e924f57f Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 19 Oct 2023 14:52:26 -0700 Subject: [PATCH 28/80] Increased test coverage to 100% for all cluster classes --- firecrown/models/cluster/abundance.py | 27 ++++--- tests/test_cluster_abundance.py | 103 +++++++++++++++++++++++++- tests/test_cluster_mass_richness.py | 13 ++-- tests/test_integrator.py | 24 ++++++ 4 files changed, 148 insertions(+), 19 deletions(-) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 833b4fa23..d9cad74f5 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Union, Callable from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl @@ -63,7 +63,9 @@ def update_ingredients(self, cosmo: Cosmology, params: ParamsMap): for kernel in self.kernels: kernel.update(params) - def comoving_volume(self, z) -> float: + def comoving_volume( + self, z: Union[float, npt.NDArray[np.float64]] + ) -> Union[float, npt.NDArray[np.float64]]: """Differential Comoving Volume at z. parameters @@ -86,10 +88,16 @@ def comoving_volume(self, z) -> float: return dV * self.sky_area_rad def mass_function( - self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] - ) -> npt.NDArray[np.float64]: + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ) -> Union[float, npt.NDArray[np.float64]]: + z = np.atleast_1d(z) + mass = np.atleast_1d(mass) + scale_factor = 1.0 / (1.0 + z) return_vals = [] + for m, a in zip(mass, scale_factor): val = self._hmf_cache.get((m, a)) if val is None: @@ -99,18 +107,17 @@ def mass_function( if len(return_vals) == 1: return return_vals[0] - return return_vals + return np.asarray(return_vals, dtype=np.float64) - def get_integrand(self, avg_mass=False, avg_redshift=False): - def integrand(*int_args): + def get_integrand( + self, avg_mass: bool = False, avg_redshift: bool = False + ) -> Callable[..., Union[float, npt.NDArray[np.float64]]]: + def integrand(*int_args) -> Union[float, npt.NDArray[np.float64]]: args_map: ArgReader = int_args[-1] z = args_map.get_integral_bounds(int_args, KernelType.z) mass = args_map.get_integral_bounds(int_args, KernelType.mass) - z = np.atleast_1d(z) - mass = np.atleast_1d(mass) - integrand = self.comoving_volume(z) * self.mass_function(mass, z) if avg_mass: integrand *= mass diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 1406068b7..5a970d53d 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -4,7 +4,7 @@ import pyccl import numpy as np from typing import List, Tuple -from firecrown.parameters import ParamsMap +from firecrown.parameters import ParamsMap, create @pytest.fixture() @@ -23,11 +23,27 @@ def __init__( integral_bounds: List[Tuple[float, float]] = None, ): super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) + self.param = create() def distribution(self, args: List[float], args_map: ArgReader): return 1.0 +class MockArgsReader(ArgReader): + def __init__(self): + super().__init__() + self.integral_bounds_idx = 0 + self.extra_args_idx = 1 + + def get_integral_bounds(self, int_args, kernel_type: KernelType): + bounds_values = int_args[self.integral_bounds_idx] + return bounds_values[:, self.integral_bounds[kernel_type.name]] + + def get_extra_args(self, int_args, kernel_type: KernelType): + extra_values = int_args[self.extra_args_idx] + return extra_values[self.extra_args[kernel_type.name]] + + def test_cluster_abundance_init(cl_abundance: ClusterAbundance): assert cl_abundance is not None assert cl_abundance.cosmo is None @@ -42,13 +58,19 @@ def test_cluster_abundance_init(cl_abundance: ClusterAbundance): def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): + mk = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(mk) + assert cl_abundance.cosmo is None + assert mk.param is None + + pmap = ParamsMap({"param": 42}) cosmo = pyccl.CosmologyVanillaLCDM() - pmap = ParamsMap() cl_abundance.update_ingredients(cosmo, pmap) assert cl_abundance.cosmo is not None assert cl_abundance.cosmo == cosmo + assert mk.param == 42 def test_cluster_sky_area(cl_abundance: ClusterAbundance): @@ -81,3 +103,80 @@ def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.analytic_kernels) == 1 assert len(cl_abundance.dirac_delta_kernels) == 1 assert len(cl_abundance.integrable_kernels) == 1 + + +def test_abundance_comoving_vol_accepts_float(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + + result = cl_abundance.comoving_volume(0.1) + assert isinstance(result, float) + assert result > 0 + + +def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + + result = cl_abundance.comoving_volume(np.linspace(0.1, 1, 10)) + assert isinstance(result, np.ndarray) + assert np.issubdtype(result.dtype, np.float64) + assert len(result) == 10 + assert np.all(result > 0) + + +def test_abundance_massfunc_accepts_float(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + + result = cl_abundance.mass_function(13, 0.1) + assert isinstance(result, float) + assert result > 0 + + +def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + + result = cl_abundance.mass_function(np.linspace(13, 17, 5), np.linspace(0.1, 1, 5)) + assert isinstance(result, np.ndarray) + assert np.issubdtype(result.dtype, np.float64) + assert len(result) == 5 + assert np.all(result > 0) + + +def test_abundance_get_integrand(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.add_kernel(MockKernel(KernelType.mass)) + + arg_reader = MockArgsReader() + arg_reader.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + integral_bounds = np.array(list(zip(mass, z))) + + integrand = cl_abundance.get_integrand() + assert callable(integrand) + result = integrand(integral_bounds, [], arg_reader) + assert isinstance(result, np.ndarray) + assert np.issubdtype(result.dtype, np.float64) + + integrand = cl_abundance.get_integrand(avg_mass=True) + assert callable(integrand) + result = integrand(integral_bounds, [], arg_reader) + assert isinstance(result, np.ndarray) + assert np.issubdtype(result.dtype, np.float64) + + integrand = cl_abundance.get_integrand(avg_redshift=True) + assert callable(integrand) + result = integrand(integral_bounds, [], arg_reader) + assert isinstance(result, np.ndarray) + assert np.issubdtype(result.dtype, np.float64) + + integrand = cl_abundance.get_integrand(avg_redshift=True, avg_mass=True) + assert callable(integrand) + result = integrand(integral_bounds, [], arg_reader) + assert isinstance(result, np.ndarray) + assert np.issubdtype(result.dtype, np.float64) diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 766053254..c1e51b3b5 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,6 +1,5 @@ import pytest import numpy as np -from firecrown.models.cluster.mass_proxy.gaussian import MassRichnessGaussian from firecrown.models.cluster.mass_proxy.murata import ( MurataBinned, MurataUnbinned, @@ -32,7 +31,7 @@ def get_extra_args(self, int_args, kernel_type: KernelType): @pytest.fixture(name="murata_binned_relation") -def fixture_cluster_mass_rich() -> MurataBinned: +def fixture_murata_binned() -> MurataBinned: """Initialize cluster object.""" mr = MurataBinned(PIVOT_MASS, PIVOT_Z) @@ -50,7 +49,7 @@ def fixture_cluster_mass_rich() -> MurataBinned: @pytest.fixture(name="murata_unbinned_relation") -def fixture_cluster_mass_rich() -> MurataUnbinned: +def fixture_murata_unbinned() -> MurataUnbinned: """Initialize cluster object.""" mr = MurataUnbinned(PIVOT_MASS, PIVOT_Z) @@ -100,7 +99,7 @@ def test_cluster_observed_mass(): def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned): - logM_array = np.linspace(7.0, 26.0, 2000) + logM_array = np.linspace(7.0, 26.0, 20) for z in np.geomspace(1.0e-18, 2.0, 20): flip = False for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): @@ -137,7 +136,7 @@ def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned def test_cluster_murata_binned_mean(murata_binned_relation: MurataBinned): - for mass in np.linspace(7.0, 26.0, 2000): + for mass in np.linspace(7.0, 26.0, 20): for z in np.geomspace(1.0e-18, 2.0, 20): test = murata_binned_relation.get_proxy_mean(mass, z) @@ -153,7 +152,7 @@ def test_cluster_murata_binned_mean(murata_binned_relation: MurataBinned): def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): - for mass in np.linspace(7.0, 26.0, 2000): + for mass in np.linspace(7.0, 26.0, 20): for z in np.geomspace(1.0e-18, 2.0, 20): test = murata_binned_relation.get_proxy_sigma(mass, z) @@ -169,7 +168,7 @@ def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): def test_cluster_murata_unbinned_distribution(murata_unbinned_relation: MurataUnbinned): - logM_array = np.linspace(7.0, 26.0, 2000) + logM_array = np.linspace(7.0, 26.0, 20) for z in np.geomspace(1.0e-18, 2.0, 20): flip = False for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): diff --git a/tests/test_integrator.py b/tests/test_integrator.py index 3d2cfdd5e..3fe3369ea 100644 --- a/tests/test_integrator.py +++ b/tests/test_integrator.py @@ -631,3 +631,27 @@ def test_scipy_get_integration_bounds_analytic_slns( KernelType.mass.name: 0, KernelType.z.name: 1, } + + +def test_scipy_integrator_integrate(): + sci = ScipyIntegrator() + + def integrand(*int_args): + x = int_args[0] + return x + + bounds = [(0, 1)] + result = sci.integrate(integrand, bounds, []) + assert result == 0.5 + + +def test_numcosmo_integrator_integrate(): + nci = NumCosmoIntegrator() + + def integrand(*int_args): + x = int_args[0] + return x + + bounds = [(0, 1)] + result = nci.integrate(integrand, bounds, []) + assert result == 0.5 From 51ce73e0d2b67e4a289abcce8ecf0470a5ad48a3 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 20 Oct 2023 10:05:53 -0700 Subject: [PATCH 29/80] Improving method names, adding typing. --- firecrown/integrator/numcosmo_integrator.py | 2 +- firecrown/integrator/scipy_integrator.py | 2 +- firecrown/models/cluster/abundance.py | 19 ++++--- firecrown/models/cluster/abundance_data.py | 20 ++++--- firecrown/models/cluster/kernel.py | 52 ++++++++++++------- .../models/cluster/mass_proxy/gaussian.py | 23 +++++--- firecrown/models/cluster/mass_proxy/murata.py | 25 ++++++--- tests/test_cluster_abundance.py | 2 +- tests/test_cluster_kernels.py | 2 +- tests/test_cluster_mass_richness.py | 2 +- tests/test_integrator.py | 16 +++--- 11 files changed, 104 insertions(+), 61 deletions(-) diff --git a/firecrown/integrator/numcosmo_integrator.py b/firecrown/integrator/numcosmo_integrator.py index 2fa981949..aba6ef0fc 100644 --- a/firecrown/integrator/numcosmo_integrator.py +++ b/firecrown/integrator/numcosmo_integrator.py @@ -15,7 +15,7 @@ def __init__(self): self.integral_bounds_idx = 0 self.extra_args_idx = 1 - def get_integral_bounds(self, int_args, kernel_type: KernelType): + def get_independent_val(self, int_args, kernel_type: KernelType): bounds_values = int_args[self.integral_bounds_idx] return bounds_values[:, self.integral_bounds[kernel_type.name]] diff --git a/firecrown/integrator/scipy_integrator.py b/firecrown/integrator/scipy_integrator.py index 56fbe8ec4..2c231b23f 100644 --- a/firecrown/integrator/scipy_integrator.py +++ b/firecrown/integrator/scipy_integrator.py @@ -87,7 +87,7 @@ def __init__(self): self.extra_args = dict() self.integral_bounds_idx = 0 - def get_integral_bounds(self, int_args, kernel_type: KernelType): + def get_independent_val(self, int_args, kernel_type: KernelType): return np.array(int_args[self.integral_bounds[kernel_type.name]]) def get_extra_args(self, int_args, kernel_type: KernelType): diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index d9cad74f5..bdeacd2d3 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -14,7 +14,7 @@ def sky_area(self) -> float: return self.sky_area_rad * (180.0 / np.pi) ** 2 @sky_area.setter - def sky_area(self, sky_area: float) -> None: + def sky_area(self, sky_area: float): self.sky_area_rad = sky_area * (np.pi / 180.0) ** 2 @property @@ -22,15 +22,15 @@ def cosmo(self) -> Cosmology: return self._cosmo @property - def analytic_kernels(self): + def analytic_kernels(self) -> List[Kernel]: return [x for x in self.kernels if x.has_analytic_sln] @property - def dirac_delta_kernels(self): + def dirac_delta_kernels(self) -> List[Kernel]: return [x for x in self.kernels if x.is_dirac_delta] @property - def integrable_kernels(self): + def integrable_kernels(self) -> List[Kernel]: return [ x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln ] @@ -76,14 +76,13 @@ def comoving_volume( """ scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) - h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) + dV = ( pyccl.physical_constants.CLIGHT_HMPC - * ((1.0 + z) ** 2) * (angular_diam_dist**2) - / self.cosmo["h"] - / h_over_h0 + * ((1.0 + z) ** 2) + / (self.cosmo["h"] * h_over_h0) ) return dV * self.sky_area_rad @@ -115,8 +114,8 @@ def get_integrand( def integrand(*int_args) -> Union[float, npt.NDArray[np.float64]]: args_map: ArgReader = int_args[-1] - z = args_map.get_integral_bounds(int_args, KernelType.z) - mass = args_map.get_integral_bounds(int_args, KernelType.mass) + z = args_map.get_independent_val(int_args, KernelType.z) + mass = args_map.get_independent_val(int_args, KernelType.mass) integrand = self.comoving_volume(z) * self.mass_function(mass, z) if avg_mass: diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 548adb2b8..2dc8f9456 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -1,6 +1,8 @@ import sacc import numpy as np from sacc.tracers import SurveyTracer +from typing import Tuple, List +import numpy.typing as npt class AbundanceData: @@ -10,7 +12,11 @@ class AbundanceData: _mass_index = 2 def __init__( - self, sacc_data: sacc.Sacc, survey_nm: str, cluster_counts, mean_log_mass + self, + sacc_data: sacc.Sacc, + survey_nm: str, + cluster_counts: bool, + mean_log_mass: bool, ): self.sacc_data = sacc_data self.cluster_counts = cluster_counts @@ -25,7 +31,7 @@ def __init__( if not isinstance(self.survey_tracer, SurveyTracer): raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") - def get_filtered_tracers(self, data_type): + def get_filtered_tracers(self, data_type: str) -> Tuple[npt.NDArray, npt.NDArray]: all_tracers = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) @@ -34,7 +40,7 @@ def get_filtered_tracers(self, data_type): filtered_tracers = all_tracers[survey_mask] return filtered_tracers, survey_mask - def get_data_and_indices(self, data_type): + def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: _, survey_mask = self.get_filtered_tracers(data_type) data_vector_list = list( self.sacc_data.get_mean(data_type=data_type)[survey_mask] @@ -44,7 +50,7 @@ def get_data_and_indices(self, data_type): ) return data_vector_list, sacc_indices_list - def validate_tracers(self, tracers_combinations, data_type): + def validate_tracers(self, tracers_combinations, data_type: str): if len(tracers_combinations) == 0: raise ValueError( f"The SACC file does not contain any tracers for the " @@ -58,13 +64,13 @@ def validate_tracers(self, tracers_combinations, data_type): "redshift argument and mass argument tracers." ) - def get_bin_limits(self, data_type): + def get_bin_limits(self, data_type: str) -> List[Tuple[float, float]]: filtered_tracers, _ = self.get_filtered_tracers(data_type) tracers = [] for _, z_tracer, mass_tracer in filtered_tracers: - z_data = self.sacc_data.get_tracer(z_tracer) - mass_data = self.sacc_data.get_tracer(mass_tracer) + z_data: sacc.BaseTracer = self.sacc_data.get_tracer(z_tracer) + mass_data: sacc.BaseTracer = self.sacc_data.get_tracer(mass_tracer) tracers.append( [(z_data.lower, z_data.upper), (mass_data.lower, mass_data.upper)] ) diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index a8f128b95..24a250c4e 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import List, Tuple - +from typing import List, Tuple, Union +import numpy.typing as npt import numpy as np from firecrown.updatable import Updatable @@ -21,11 +21,15 @@ def __init__(self): self.extra_args = dict() @abstractmethod - def get_integral_bounds(self, int_args, kernel_type: KernelType): + def get_independent_val( + self, integral_args: List[float], kernel_type: KernelType + ) -> Union[float, npt.NDArray[np.float64]]: """Returns the current differential value for KernelType""" @abstractmethod - def get_extra_args(self, int_args, kernel_type: KernelType): + def get_extra_args( + self, integral_args: List[float], kernel_type: KernelType + ) -> Union[float, Tuple[float, float]]: """Returns the extra arguments passed into the integral for KernelType""" @@ -33,8 +37,8 @@ class Kernel(Updatable, ABC): def __init__( self, kernel_type: KernelType, - is_dirac_delta=False, - has_analytic_sln=False, + is_dirac_delta: bool = False, + has_analytic_sln: bool = False, integral_bounds: List[Tuple[float, float]] = None, ): super().__init__() @@ -43,7 +47,9 @@ def __init__( self.kernel_type = kernel_type self.has_analytic_sln = has_analytic_sln - def distribution(self, args: List[float], args_map: ArgReader): + def distribution( + self, args: List[float], arg_reader: ArgReader + ) -> Union[float, npt.NDArray[np.float64]]: """The functional form of the distribution or spread of this kernel""" @@ -51,9 +57,11 @@ class Completeness(Kernel): def __init__(self): super().__init__(KernelType.completeness) - def distribution(self, args: List[float], args_map: ArgReader): - mass = args_map.get_integral_bounds(args, KernelType.mass) - z = args_map.get_integral_bounds(args, KernelType.z) + def distribution( + self, args: List[float], arg_reader: ArgReader + ) -> Union[float, npt.NDArray[np.float64]]: + mass = arg_reader.get_independent_val(args, KernelType.mass) + z = arg_reader.get_independent_val(args, KernelType.z) a_nc = 1.1321 b_nc = 0.7751 @@ -69,9 +77,11 @@ class Purity(Kernel): def __init__(self): super().__init__(KernelType.purity) - def distribution(self, args: List[float], args_index_map: ArgReader): - mass_proxy = args_index_map.get_integral_bounds(args, KernelType.mass_proxy) - z = args_index_map.get_integral_bounds(args, KernelType.z) + def distribution( + self, args: List[float], arg_reader: ArgReader + ) -> Union[float, npt.NDArray[np.float64]]: + mass_proxy = arg_reader.get_independent_val(args, KernelType.mass_proxy) + z = arg_reader.get_independent_val(args, KernelType.z) a_nc = np.log(10) * 0.8612 b_nc = np.log(10) * 0.3527 @@ -92,7 +102,9 @@ class TrueMass(Kernel): def __init__(self): super().__init__(KernelType.mass_proxy, True) - def distribution(self, args: List[float], args_map: ArgReader): + def distribution( + self, args: List[float], arg_reader: ArgReader + ) -> Union[float, npt.NDArray[np.float64]]: return 1.0 @@ -100,7 +112,9 @@ class SpectroscopicRedshift(Kernel): def __init__(self): super().__init__(KernelType.z_proxy, True) - def distribution(self, args: List[float], args_map: ArgReader): + def distribution( + self, args: List[float], arg_reader: ArgReader + ) -> Union[float, npt.NDArray[np.float64]]: return 1.0 @@ -109,9 +123,11 @@ def __init__(self): super().__init__(KernelType.z_proxy) self.sigma_0 = 0.05 - def distribution(self, args: List[float], args_map: ArgReader): - z = args_map.get_integral_bounds(args, KernelType.z) - z_proxy = args_map.get_integral_bounds(args, KernelType.z_proxy) + def distribution( + self, args: List[float], arg_reader: ArgReader + ) -> Union[float, npt.NDArray[np.float64]]: + z = arg_reader.get_independent_val(args, KernelType.z) + z_proxy = arg_reader.get_independent_val(args, KernelType.z_proxy) sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) diff --git a/firecrown/models/cluster/mass_proxy/gaussian.py b/firecrown/models/cluster/mass_proxy/gaussian.py index 4ea01938a..a9cda2430 100644 --- a/firecrown/models/cluster/mass_proxy/gaussian.py +++ b/firecrown/models/cluster/mass_proxy/gaussian.py @@ -1,19 +1,28 @@ -from typing import List +from typing import List, Union +import numpy.typing as npt import numpy as np from scipy import special from firecrown.models.cluster.kernel import ArgReader, Kernel, KernelType class MassRichnessGaussian(Kernel): - def get_proxy_mean(self, mass, z): + def get_proxy_mean( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): """Return observed quantity corrected by redshift and mass.""" - def get_proxy_sigma(self, mass, z): + def get_proxy_sigma( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): """Return observed scatter corrected by redshift and mass.""" def _distribution_binned(self, args: List[float], args_map: ArgReader): - mass = args_map.get_integral_bounds(args, KernelType.mass) - z = args_map.get_integral_bounds(args, KernelType.z) + mass = args_map.get_independent_val(args, KernelType.mass) + z = args_map.get_independent_val(args, KernelType.z) mass_proxy_limits = args_map.get_extra_args(args, self.kernel_type) proxy_mean = self.get_proxy_mean(mass, z) @@ -40,8 +49,8 @@ def _distribution_binned(self, args: List[float], args_map: ArgReader): return return_vals def _distribution_unbinned(self, args: List[float], args_map: ArgReader): - mass = args_map.get_integral_bounds(args, KernelType.mass) - z = args_map.get_integral_bounds(args, KernelType.z) + mass = args_map.get_independent_val(args, KernelType.mass) + z = args_map.get_independent_val(args, KernelType.z) mass_proxy = args_map.get_extra_args(args, self.kernel_type) proxy_mean = self.get_proxy_mean(mass, z) diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py index d656f21eb..7440a3127 100644 --- a/firecrown/models/cluster/mass_proxy/murata.py +++ b/firecrown/models/cluster/mass_proxy/murata.py @@ -1,6 +1,7 @@ -from typing import List, Tuple +from typing import List, Tuple, Union import numpy as np +import numpy.typing as npt from firecrown import parameters from firecrown.models.cluster.kernel import KernelType @@ -8,7 +9,11 @@ def _observed_value( - p: Tuple[float, float, float], mass, z, pivot_mass, log1p_pivot_redshift + p: Tuple[float, float, float], + mass: npt.NDArray[np.float64], + z: float, + pivot_mass: float, + log1p_pivot_redshift: float, ): """Return observed quantity corrected by redshift and mass.""" @@ -22,8 +27,8 @@ def _observed_value( class MurataCore(MassRichnessGaussian): def __init__( self, - pivot_mass, - pivot_redshift, + pivot_mass: float, + pivot_redshift: float, integral_bounds: List[Tuple[float, float]] = None, ): super().__init__(KernelType.mass_proxy, False, True, integral_bounds) @@ -42,7 +47,11 @@ def __init__( # Verify this gets called last or first - def get_proxy_mean(self, mass, z): + def get_proxy_mean( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): """Return observed quantity corrected by redshift and mass.""" return _observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), @@ -52,7 +61,11 @@ def get_proxy_mean(self, mass, z): self.log1p_pivot_redshift, ) - def get_proxy_sigma(self, mass, z): + def get_proxy_sigma( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): """Return observed scatter corrected by redshift and mass.""" return _observed_value( (self.sigma_p0, self.sigma_p1, self.sigma_p2), diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 5a970d53d..bf09f0538 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -35,7 +35,7 @@ def __init__(self): self.integral_bounds_idx = 0 self.extra_args_idx = 1 - def get_integral_bounds(self, int_args, kernel_type: KernelType): + def get_independent_val(self, int_args, kernel_type: KernelType): bounds_values = int_args[self.integral_bounds_idx] return bounds_values[:, self.integral_bounds[kernel_type.name]] diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 74ff9a1e1..106f95d01 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -18,7 +18,7 @@ def __init__(self): self.integral_bounds_idx = 0 self.extra_args_idx = 1 - def get_integral_bounds(self, int_args, kernel_type: KernelType): + def get_independent_val(self, int_args, kernel_type: KernelType): bounds_values = int_args[self.integral_bounds_idx] return bounds_values[:, self.integral_bounds[kernel_type.name]] diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index c1e51b3b5..38be8aeba 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -21,7 +21,7 @@ def __init__(self): self.integral_bounds_idx = 0 self.extra_args_idx = 1 - def get_integral_bounds(self, int_args, kernel_type: KernelType): + def get_independent_val(self, int_args, kernel_type: KernelType): bounds_values = int_args[self.integral_bounds_idx] return bounds_values[:, self.integral_bounds[kernel_type.name]] diff --git a/tests/test_integrator.py b/tests/test_integrator.py index 3fe3369ea..79133accf 100644 --- a/tests/test_integrator.py +++ b/tests/test_integrator.py @@ -72,13 +72,13 @@ def test_numcosmo_argreader_integral_bounds(): integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) int_args = [integral_bounds] - assert (mass == arg_reader.get_integral_bounds(int_args, KernelType.mass)).all() - assert (z == arg_reader.get_integral_bounds(int_args, KernelType.z)).all() + assert (mass == arg_reader.get_independent_val(int_args, KernelType.mass)).all() + assert (z == arg_reader.get_independent_val(int_args, KernelType.z)).all() assert ( - z_proxy == arg_reader.get_integral_bounds(int_args, KernelType.z_proxy) + z_proxy == arg_reader.get_independent_val(int_args, KernelType.z_proxy) ).all() assert ( - mass_proxy == arg_reader.get_integral_bounds(int_args, KernelType.mass_proxy) + mass_proxy == arg_reader.get_independent_val(int_args, KernelType.mass_proxy) ).all() @@ -124,10 +124,10 @@ def test_scipy_argreader_integral_bounds(): for m_i, z_i, zp_i, mp_i in list(zip(mass, z, z_proxy, mass_proxy)): int_args = [m_i, z_i, zp_i, mp_i] - assert m_i == arg_reader.get_integral_bounds(int_args, KernelType.mass) - assert z_i == arg_reader.get_integral_bounds(int_args, KernelType.z) - assert zp_i == arg_reader.get_integral_bounds(int_args, KernelType.z_proxy) - assert mp_i == arg_reader.get_integral_bounds(int_args, KernelType.mass_proxy) + assert m_i == arg_reader.get_independent_val(int_args, KernelType.mass) + assert z_i == arg_reader.get_independent_val(int_args, KernelType.z) + assert zp_i == arg_reader.get_independent_val(int_args, KernelType.z_proxy) + assert mp_i == arg_reader.get_independent_val(int_args, KernelType.mass_proxy) def test_create_scipy_argreader(): From 96e9d0f71177fb0a5377c71ef2a7048440a23677 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 23 Oct 2023 08:29:59 -0700 Subject: [PATCH 30/80] Fixing other unit tests by making params optional to modeling tools update. --- firecrown/modeling_tools.py | 4 +++- firecrown/models/cluster/abundance.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index 0633c7c8f..87cb6cffd 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -57,7 +57,9 @@ def has_pk(self, name: str) -> bool: return False return True - def prepare(self, ccl_cosmo: pyccl.Cosmology, params: ParamsMap) -> None: + def prepare( + self, ccl_cosmo: pyccl.Cosmology, params: Optional[ParamsMap] = None + ) -> None: """Prepare the Cosmology for use in likelihoods. This method will prepare the ModelingTools for use in likelihoods. This diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index bdeacd2d3..2c541c01c 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,4 +1,4 @@ -from typing import List, Union, Callable +from typing import List, Union, Callable, Optional from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl @@ -57,9 +57,12 @@ def __init__( def add_kernel(self, kernel: Kernel): self.kernels.append(kernel) - def update_ingredients(self, cosmo: Cosmology, params: ParamsMap): + def update_ingredients(self, cosmo: Cosmology, params: Optional[ParamsMap] = None): self._cosmo = cosmo self._hmf_cache = {} + if params is None: + return + for kernel in self.kernels: kernel.update(params) From 849aa76f7b983228f8c4714e6e42bfe00e680d0e Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 23 Oct 2023 08:54:09 -0700 Subject: [PATCH 31/80] Mypy type fixes. --- .../statistic/binned_cluster_number_counts.py | 2 ++ firecrown/models/cluster/abundance.py | 4 ++-- firecrown/models/cluster/abundance_data.py | 2 +- firecrown/models/cluster/kernel.py | 4 ++-- firecrown/models/cluster/mass_proxy/murata.py | 8 ++++---- tests/test_cluster_abundance.py | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index f2b9b3395..36c5b351a 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -72,6 +72,8 @@ def get_data_vector(self) -> DataVector: return self.data_vector def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: + assert tools.cluster_abundance is not None + theory_vector_list = [] cluster_counts = [] cluster_masses = [] diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 2c541c01c..29d93ed79 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,4 +1,4 @@ -from typing import List, Union, Callable, Optional +from typing import List, Union, Callable, Optional, Dict, Tuple from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl @@ -51,7 +51,7 @@ def __init__( self.min_z = min_z self.max_z = max_z self.sky_area = sky_area - self._hmf_cache = {} + self._hmf_cache: Dict[Tuple[float, float], float] = {} self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel): diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 2dc8f9456..148ce9150 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -64,7 +64,7 @@ def validate_tracers(self, tracers_combinations, data_type: str): "redshift argument and mass argument tracers." ) - def get_bin_limits(self, data_type: str) -> List[Tuple[float, float]]: + def get_bin_limits(self, data_type: str) -> List[List[Tuple[float, float]]]: filtered_tracers, _ = self.get_filtered_tracers(data_type) tracers = [] diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 24a250c4e..26dd0a685 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import List, Tuple, Union +from typing import List, Tuple, Union, Optional import numpy.typing as npt import numpy as np from firecrown.updatable import Updatable @@ -39,7 +39,7 @@ def __init__( kernel_type: KernelType, is_dirac_delta: bool = False, has_analytic_sln: bool = False, - integral_bounds: List[Tuple[float, float]] = None, + integral_bounds: Optional[List[Tuple[float, float]]] = None, ): super().__init__() self.integral_bounds = integral_bounds diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py index 7440a3127..fe261fd86 100644 --- a/firecrown/models/cluster/mass_proxy/murata.py +++ b/firecrown/models/cluster/mass_proxy/murata.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Union +from typing import List, Tuple, Union, Optional import numpy as np import numpy.typing as npt @@ -10,8 +10,8 @@ def _observed_value( p: Tuple[float, float, float], - mass: npt.NDArray[np.float64], - z: float, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], pivot_mass: float, log1p_pivot_redshift: float, ): @@ -29,7 +29,7 @@ def __init__( self, pivot_mass: float, pivot_redshift: float, - integral_bounds: List[Tuple[float, float]] = None, + integral_bounds: Optional[List[Tuple[float, float]]] = None, ): super().__init__(KernelType.mass_proxy, False, True, integral_bounds) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index bf09f0538..a0328b767 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -3,7 +3,7 @@ from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader import pyccl import numpy as np -from typing import List, Tuple +from typing import List, Tuple, Optional from firecrown.parameters import ParamsMap, create @@ -20,7 +20,7 @@ def __init__( kernel_type: KernelType, is_dirac_delta: bool = False, has_analytic_sln: bool = False, - integral_bounds: List[Tuple[float, float]] = None, + integral_bounds: Optional[List[Tuple[float, float]]] = None, ): super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) self.param = create() From b6b7b2578841314a34cdbf2830513585d41031e7 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 25 Oct 2023 13:48:48 -0700 Subject: [PATCH 32/80] Fixed all mypy complaints. --- .../cluster_redshift_richness.py | 8 +- firecrown/integrator/__init__.py | 1 - .../statistic/binned_cluster_number_counts.py | 2 +- firecrown/models/cluster/abundance.py | 64 ++++-- .../{mass_proxy => integrator}/__init__.py | 0 .../cluster}/integrator/integrator.py | 10 +- .../integrator/numcosmo_integrator.py | 31 +-- .../cluster}/integrator/scipy_integrator.py | 17 +- firecrown/models/cluster/kernel.py | 51 +++-- firecrown/models/cluster/mass_proxy.py | 195 ++++++++++++++++++ .../models/cluster/mass_proxy/gaussian.py | 61 ------ firecrown/models/cluster/mass_proxy/murata.py | 86 -------- tests/test_cluster.py | 167 --------------- tests/test_cluster_abundance.py | 7 +- ...tegrator.py => test_cluster_integrator.py} | 21 +- tests/test_cluster_kernels.py | 13 +- tests/test_cluster_mass_richness.py | 12 +- 17 files changed, 346 insertions(+), 400 deletions(-) delete mode 100644 firecrown/integrator/__init__.py rename firecrown/models/cluster/{mass_proxy => integrator}/__init__.py (100%) rename firecrown/{ => models/cluster}/integrator/integrator.py (73%) rename firecrown/{ => models/cluster}/integrator/numcosmo_integrator.py (83%) rename firecrown/{ => models/cluster}/integrator/scipy_integrator.py (87%) create mode 100644 firecrown/models/cluster/mass_proxy.py delete mode 100644 firecrown/models/cluster/mass_proxy/gaussian.py delete mode 100644 firecrown/models/cluster/mass_proxy/murata.py delete mode 100644 tests/test_cluster.py rename tests/{test_integrator.py => test_cluster_integrator.py} (97%) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 788d4d898..adb1dd2a3 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -5,8 +5,8 @@ import pyccl as ccl import sacc -from firecrown.integrator.numcosmo_integrator import NumCosmoIntegrator -from firecrown.integrator.scipy_integrator import ScipyIntegrator +from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator +from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( BinnedClusterNumberCounts, @@ -43,8 +43,8 @@ def get_cluster_abundance(sky_area): # completeness_kernel = Completeness() # cluster_abundance.add_kernel(completeness_kernel) - # # purity_kernel = Purity() - # cluster_abundance.add_kernel(purity_kernel) + purity_kernel = Purity() + cluster_abundance.add_kernel(purity_kernel) return cluster_abundance diff --git a/firecrown/integrator/__init__.py b/firecrown/integrator/__init__.py deleted file mode 100644 index e0fccee8a..000000000 --- a/firecrown/integrator/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .integrator import Integrator diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 36c5b351a..b253bf06a 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -2,7 +2,7 @@ from typing import List, Optional import sacc -from firecrown.integrator import Integrator +from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.models.cluster.abundance_data import AbundanceData from firecrown.likelihood.gauss_family.statistic.statistic import ( Statistic, diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 29d93ed79..a0ccb867f 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,4 +1,4 @@ -from typing import List, Union, Callable, Optional, Dict, Tuple +from typing import List, Union, Callable, Optional, Dict, Tuple, Any from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl @@ -6,6 +6,8 @@ import numpy as np from firecrown.parameters import ParamsMap import numpy.typing as npt +from functools import singledispatchmethod +import pdb class ClusterAbundance(object): @@ -14,7 +16,7 @@ def sky_area(self) -> float: return self.sky_area_rad * (180.0 / np.pi) ** 2 @sky_area.setter - def sky_area(self, sky_area: float): + def sky_area(self, sky_area: float) -> None: self.sky_area_rad = sky_area * (np.pi / 180.0) ** 2 @property @@ -43,7 +45,7 @@ def __init__( max_z: float, halo_mass_function: pyccl.halos.MassFunc, sky_area: float, - ): + ) -> None: self.kernels: List[Kernel] = [] self.halo_mass_function = halo_mass_function self.min_mass = min_mass @@ -54,10 +56,12 @@ def __init__( self._hmf_cache: Dict[Tuple[float, float], float] = {} self._cosmo: Cosmology = None - def add_kernel(self, kernel: Kernel): + def add_kernel(self, kernel: Kernel) -> None: self.kernels.append(kernel) - def update_ingredients(self, cosmo: Cosmology, params: Optional[ParamsMap] = None): + def update_ingredients( + self, cosmo: Cosmology, params: Optional[ParamsMap] = None + ) -> None: self._cosmo = cosmo self._hmf_cache = {} if params is None: @@ -66,6 +70,7 @@ def update_ingredients(self, cosmo: Cosmology, params: Optional[ParamsMap] = Non for kernel in self.kernels: kernel.update(params) + @singledispatchmethod def comoving_volume( self, z: Union[float, npt.NDArray[np.float64]] ) -> Union[float, npt.NDArray[np.float64]]: @@ -77,6 +82,10 @@ def comoving_volume( :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). """ + raise ValueError("Unsupported type for z:", type(z)) + + @comoving_volume.register(np.ndarray) + def _(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) @@ -87,16 +96,37 @@ def comoving_volume( * ((1.0 + z) ** 2) / (self.cosmo["h"] * h_over_h0) ) + assert isinstance(dV, np.ndarray) return dV * self.sky_area_rad + @comoving_volume.register(float) + def _(self, z: float) -> float: + z_arr = np.atleast_1d(z) + vol = self.comoving_volume(z_arr) + assert isinstance(vol, np.ndarray) + return vol.item() + + @singledispatchmethod def mass_function( self, mass: Union[float, npt.NDArray[np.float64]], z: Union[float, npt.NDArray[np.float64]], ) -> Union[float, npt.NDArray[np.float64]]: - z = np.atleast_1d(z) - mass = np.atleast_1d(mass) - + """Halo Mass Function at mass and z.""" + raise ValueError("Unsupported type for either mass or z:", type(mass), type(z)) + + @mass_function.register(float) + def _(self, mass: float, z: float) -> float: + z_arr = np.atleast_1d(z) + mass_arr = np.atleast_1d(mass) + mf = self.mass_function(mass_arr, z_arr) + assert isinstance(mf, np.ndarray) + return mf.item() + + @mass_function.register(np.ndarray) + def _( + self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] + ) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) return_vals = [] @@ -107,18 +137,18 @@ def mass_function( self._hmf_cache[(m, a)] = val return_vals.append(val) - if len(return_vals) == 1: - return return_vals[0] return np.asarray(return_vals, dtype=np.float64) def get_integrand( self, avg_mass: bool = False, avg_redshift: bool = False - ) -> Callable[..., Union[float, npt.NDArray[np.float64]]]: - def integrand(*int_args) -> Union[float, npt.NDArray[np.float64]]: - args_map: ArgReader = int_args[-1] + ) -> Callable[..., npt.NDArray[np.float64]]: + def integrand(*int_args) -> npt.NDArray[np.float64]: + args_map = int_args[-1] + assert isinstance(args_map, ArgReader) + values_for_integral = int_args[:1] - z = args_map.get_independent_val(int_args, KernelType.z) - mass = args_map.get_independent_val(int_args, KernelType.mass) + z = args_map.get_independent_val(values_for_integral, KernelType.z) + mass = args_map.get_independent_val(values_for_integral, KernelType.mass) integrand = self.comoving_volume(z) * self.mass_function(mass, z) if avg_mass: @@ -129,7 +159,7 @@ def integrand(*int_args) -> Union[float, npt.NDArray[np.float64]]: for kernel in self.kernels: # Think of overhead here, if its worth it integrand *= kernel.distribution(int_args, args_map) - - return integrand + return_val = np.atleast_1d(integrand) + return return_val return integrand diff --git a/firecrown/models/cluster/mass_proxy/__init__.py b/firecrown/models/cluster/integrator/__init__.py similarity index 100% rename from firecrown/models/cluster/mass_proxy/__init__.py rename to firecrown/models/cluster/integrator/__init__.py diff --git a/firecrown/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py similarity index 73% rename from firecrown/integrator/integrator.py rename to firecrown/models/cluster/integrator/integrator.py index 557639e80..19dda311e 100644 --- a/firecrown/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod -from firecrown.models.cluster.kernel import ArgReader from firecrown.models.cluster.abundance import ClusterAbundance +from typing import Tuple, List, Any +from firecrown.models.cluster.kernel import ArgReader class Integrator(ABC): @@ -15,6 +16,9 @@ def integrate(self, integrand, bounds, extra_args): @abstractmethod def get_integration_bounds( - self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits - ): + self, + cl_abundance: ClusterAbundance, + z_proxy_limits: Tuple[float, float], + mass_proxy_limits: Tuple[float, float], + ) -> Tuple[Any, ...]: """Extract the limits of integration and extra arguments for the integral""" diff --git a/firecrown/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py similarity index 83% rename from firecrown/integrator/numcosmo_integrator.py rename to firecrown/models/cluster/integrator/numcosmo_integrator.py index aba6ef0fc..52d24fb97 100644 --- a/firecrown/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -1,13 +1,14 @@ from numcosmo_py import Ncm -from typing import Tuple +from typing import Tuple, List, Union, Callable, Any import numpy as np -from firecrown.models.cluster.kernel import ArgReader, KernelType +import numpy.typing as npt +from firecrown.models.cluster.kernel import KernelType, ArgReader from firecrown.models.cluster.abundance import ClusterAbundance -from firecrown.integrator import Integrator +from firecrown.models.cluster.integrator.integrator import Integrator class NumCosmoArgReader(ArgReader): - def __init__(self): + def __init__(self) -> None: super().__init__() self.integral_bounds = dict() self.extra_args = dict() @@ -15,7 +16,9 @@ def __init__(self): self.integral_bounds_idx = 0 self.extra_args_idx = 1 - def get_independent_val(self, int_args, kernel_type: KernelType): + def get_independent_val( + self, int_args, kernel_type: KernelType + ) -> Union[float, npt.NDArray[np.float64]]: bounds_values = int_args[self.integral_bounds_idx] return bounds_values[:, self.integral_bounds[kernel_type.name]] @@ -25,15 +28,18 @@ def get_extra_args(self, int_args, kernel_type: KernelType): class NumCosmoIntegrator(Integrator): - def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): + def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12) -> None: super().__init__() self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance self.arg_reader = NumCosmoArgReader() def get_integration_bounds( - self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits - ): + self, + cl_abundance: ClusterAbundance, + z_proxy_limits: Tuple[float, float], + mass_proxy_limits: Tuple[float, float], + ) -> Tuple[Any, ...]: self.arg_reader.integral_bounds = { KernelType.mass.name: 0, KernelType.z.name: 1, @@ -64,7 +70,7 @@ def get_integration_bounds( integral_bounds.append(mass_proxy_limits) if kernel.integral_bounds is not None: - integral_bounds.append(kernel.integral_bounds) + integral_bounds.append(*kernel.integral_bounds) # Lastly, don't integrate any kernels with an analytic solution # This means we pass in their limits as extra arguments to the integrator @@ -79,11 +85,11 @@ def get_integration_bounds( extra_args.append(mass_proxy_limits) if kernel.integral_bounds is not None: - extra_args.append(kernel.integral_bounds) + extra_args.append(*kernel.integral_bounds) return integral_bounds, extra_args - def integrate(self, integrand, bounds, extra_args): + def integrate(self, integrand: Callable, bounds, extra_args): Ncm.cfg_init() int_nd = CountsIntegralND( len(bounds), @@ -106,13 +112,12 @@ class CountsIntegralND(Ncm.IntegralND): """Integral subclass used by the ClusterAbundance to compute the integrals using numcosmo.""" - def __init__(self, dim, fun, *args): + def __init__(self, dim, fun, *args) -> None: super().__init__() self.dim = dim self.fun = fun self.args = args - # pylint: disable-next=arguments-differ def do_get_dimensions(self) -> Tuple[int, int]: """Get number of dimensions.""" return self.dim, 1 diff --git a/firecrown/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py similarity index 87% rename from firecrown/integrator/scipy_integrator.py rename to firecrown/models/cluster/integrator/scipy_integrator.py index 2c231b23f..4fe00ebbc 100644 --- a/firecrown/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -1,7 +1,7 @@ from scipy.integrate import nquad - -from firecrown.integrator import Integrator -from firecrown.models.cluster.kernel import ArgReader, KernelType +from typing import Tuple, Any +from firecrown.models.cluster.integrator.integrator import Integrator +from firecrown.models.cluster.kernel import KernelType, ArgReader from firecrown.models.cluster.abundance import ClusterAbundance import numpy as np @@ -14,8 +14,11 @@ def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): self.arg_reader = ScipyArgReader() def get_integration_bounds( - self, cl_abundance: ClusterAbundance, z_proxy_limits, mass_proxy_limits - ): + self, + cl_abundance: ClusterAbundance, + z_proxy_limits: Tuple[float, float], + mass_proxy_limits: Tuple[float, float], + ) -> Tuple[Any, ...]: self.arg_reader.integral_bounds = { KernelType.mass.name: 0, KernelType.z.name: 1, @@ -46,7 +49,7 @@ def get_integration_bounds( integral_bounds.append(mass_proxy_limits) if kernel.integral_bounds is not None: - integral_bounds.append(kernel.integral_bounds) + integral_bounds.append(*kernel.integral_bounds) # Lastly, don't integrate any kernels with an analytic solution # This means we pass in their limits as extra arguments to the integrator @@ -62,7 +65,7 @@ def get_integration_bounds( extra_args.append(mass_proxy_limits) if kernel.integral_bounds is not None: - extra_args.append(kernel.integral_bounds) + extra_args.append(*kernel.integral_bounds) return integral_bounds, extra_args diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 26dd0a685..542adb01e 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import List, Tuple, Union, Optional +from typing import List, Tuple, Union, Optional, Dict, Any import numpy.typing as npt import numpy as np from firecrown.updatable import Updatable @@ -16,19 +16,23 @@ class KernelType(Enum): class ArgReader(ABC): - def __init__(self): - self.integral_bounds = dict() - self.extra_args = dict() + def __init__(self) -> None: + self.integral_bounds: Dict[str, int] = dict() + self.extra_args: Dict[str, int] = dict() @abstractmethod def get_independent_val( - self, integral_args: List[float], kernel_type: KernelType + self, + integral_args: Tuple[Any, ...], + kernel_type: KernelType, ) -> Union[float, npt.NDArray[np.float64]]: """Returns the current differential value for KernelType""" @abstractmethod def get_extra_args( - self, integral_args: List[float], kernel_type: KernelType + self, + integral_args: Tuple[Any, ...], + kernel_type: KernelType, ) -> Union[float, Tuple[float, float]]: """Returns the extra arguments passed into the integral for KernelType""" @@ -47,18 +51,23 @@ def __init__( self.kernel_type = kernel_type self.has_analytic_sln = has_analytic_sln + @abstractmethod def distribution( - self, args: List[float], arg_reader: ArgReader + self, + args: Tuple[Any, ...], + arg_reader: ArgReader, ) -> Union[float, npt.NDArray[np.float64]]: """The functional form of the distribution or spread of this kernel""" class Completeness(Kernel): - def __init__(self): + def __init__(self) -> None: super().__init__(KernelType.completeness) def distribution( - self, args: List[float], arg_reader: ArgReader + self, + args: Tuple[Any, ...], + arg_reader: ArgReader, ) -> Union[float, npt.NDArray[np.float64]]: mass = arg_reader.get_independent_val(args, KernelType.mass) z = arg_reader.get_independent_val(args, KernelType.z) @@ -74,11 +83,13 @@ def distribution( class Purity(Kernel): - def __init__(self): + def __init__(self) -> None: super().__init__(KernelType.purity) def distribution( - self, args: List[float], arg_reader: ArgReader + self, + args: Tuple[Any, ...], + arg_reader: ArgReader, ) -> Union[float, npt.NDArray[np.float64]]: mass_proxy = arg_reader.get_independent_val(args, KernelType.mass_proxy) z = arg_reader.get_independent_val(args, KernelType.z) @@ -99,32 +110,38 @@ def distribution( class TrueMass(Kernel): - def __init__(self): + def __init__(self) -> None: super().__init__(KernelType.mass_proxy, True) def distribution( - self, args: List[float], arg_reader: ArgReader + self, + args: Tuple[Any, ...], + arg_reader: ArgReader, ) -> Union[float, npt.NDArray[np.float64]]: return 1.0 class SpectroscopicRedshift(Kernel): - def __init__(self): + def __init__(self) -> None: super().__init__(KernelType.z_proxy, True) def distribution( - self, args: List[float], arg_reader: ArgReader + self, + args: Tuple[Any, ...], + arg_reader: ArgReader, ) -> Union[float, npt.NDArray[np.float64]]: return 1.0 class DESY1PhotometricRedshift(Kernel): - def __init__(self): + def __init__(self) -> None: super().__init__(KernelType.z_proxy) self.sigma_0 = 0.05 def distribution( - self, args: List[float], arg_reader: ArgReader + self, + args: Tuple[Any, ...], + arg_reader: ArgReader, ) -> Union[float, npt.NDArray[np.float64]]: z = arg_reader.get_independent_val(args, KernelType.z) z_proxy = arg_reader.get_independent_val(args, KernelType.z_proxy) diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py new file mode 100644 index 000000000..7959d6733 --- /dev/null +++ b/firecrown/models/cluster/mass_proxy.py @@ -0,0 +1,195 @@ +from typing import List, Tuple, Union, Optional, Any + +import numpy as np +import numpy.typing as npt + +from firecrown import parameters +from scipy import special +from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader +from abc import abstractmethod + + +class MassRichnessGaussian(Kernel): + @staticmethod + def observed_value( + p: Tuple[float, float, float], + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + pivot_mass: float, + log1p_pivot_redshift: float, + ) -> Union[float, npt.NDArray[np.float64]]: + """Return observed quantity corrected by redshift and mass.""" + + ln_mass = mass * np.log(10) + delta_ln_mass = ln_mass - pivot_mass + delta_z = np.log1p(z) - log1p_pivot_redshift + + return p[0] + p[1] * delta_ln_mass + p[2] * delta_z + + @abstractmethod + def get_proxy_mean( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ) -> Union[float, npt.NDArray[np.float64]]: + """Return observed quantity corrected by redshift and mass.""" + + @abstractmethod + def get_proxy_sigma( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ) -> Union[float, npt.NDArray[np.float64]]: + """Return observed scatter corrected by redshift and mass.""" + + def _distribution_binned(self, args: Tuple[Any, ...], args_map: ArgReader): + mass = args_map.get_independent_val(args, KernelType.mass) + z = args_map.get_independent_val(args, KernelType.z) + mass_proxy_limits = args_map.get_extra_args(args, self.kernel_type) + assert isinstance(mass_proxy_limits, tuple) + + proxy_mean = self.get_proxy_mean(mass, z) + proxy_sigma = self.get_proxy_sigma(mass, z) + + x_min = (proxy_mean - mass_proxy_limits[0] * np.log(10.0)) / ( + np.sqrt(2.0) * proxy_sigma + ) + x_max = (proxy_mean - mass_proxy_limits[1] * np.log(10.0)) / ( + np.sqrt(2.0) * proxy_sigma + ) + + return_vals = np.empty_like(x_min) + mask1 = (x_max > 3.0) | (x_min < -3.0) + mask2 = ~mask1 + + return_vals[mask1] = ( + -(special.erfc(x_min[mask1]) - special.erfc(x_max[mask1])) / 2.0 + ) + return_vals[mask2] = ( + special.erf(x_min[mask2]) - special.erf(x_max[mask2]) + ) / 2.0 + + return return_vals + + def _distribution_unbinned(self, args: Tuple[Any, ...], args_map: ArgReader): + mass = args_map.get_independent_val(args, KernelType.mass) + z = args_map.get_independent_val(args, KernelType.z) + mass_proxy = args_map.get_extra_args(args, self.kernel_type) + assert isinstance(mass_proxy, float) + + proxy_mean = self.get_proxy_mean(mass, z) + proxy_sigma = self.get_proxy_sigma(mass, z) + + return np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( + 2 * np.pi * proxy_sigma + ) + + +class MurataBinned(MassRichnessGaussian): + def __init__( + self, + pivot_mass: float, + pivot_redshift: float, + integral_bounds: Optional[List[Tuple[float, float]]] = None, + ): + super().__init__(KernelType.mass_proxy, False, True, integral_bounds) + + self.pivot_redshift = pivot_redshift + self.pivot_mass = pivot_mass * np.log(10.0) # ln(M) + self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) + + # Updatable parameters + self.mu_p0 = parameters.create() + self.mu_p1 = parameters.create() + self.mu_p2 = parameters.create() + self.sigma_p0 = parameters.create() + self.sigma_p1 = parameters.create() + self.sigma_p2 = parameters.create() + + # Verify this gets called last or first + + def get_proxy_mean( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): + """Return observed quantity corrected by redshift and mass.""" + return MassRichnessGaussian.observed_value( + (self.mu_p0, self.mu_p1, self.mu_p2), + mass, + z, + self.pivot_mass, + self.log1p_pivot_redshift, + ) + + def get_proxy_sigma( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): + """Return observed scatter corrected by redshift and mass.""" + return MassRichnessGaussian.observed_value( + (self.sigma_p0, self.sigma_p1, self.sigma_p2), + mass, + z, + self.pivot_mass, + self.log1p_pivot_redshift, + ) + + def distribution(self, args, args_map): + return self._distribution_binned(args, args_map) + + +class MurataUnbinned(MassRichnessGaussian): + def __init__( + self, + pivot_mass: float, + pivot_redshift: float, + integral_bounds: Optional[List[Tuple[float, float]]] = None, + ): + super().__init__(KernelType.mass_proxy, False, True, integral_bounds) + + self.pivot_redshift = pivot_redshift + self.pivot_mass = pivot_mass * np.log(10.0) # ln(M) + self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) + + # Updatable parameters + self.mu_p0 = parameters.create() + self.mu_p1 = parameters.create() + self.mu_p2 = parameters.create() + self.sigma_p0 = parameters.create() + self.sigma_p1 = parameters.create() + self.sigma_p2 = parameters.create() + + # Verify this gets called last or first + + def get_proxy_mean( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): + """Return observed quantity corrected by redshift and mass.""" + return MassRichnessGaussian.observed_value( + (self.mu_p0, self.mu_p1, self.mu_p2), + mass, + z, + self.pivot_mass, + self.log1p_pivot_redshift, + ) + + def get_proxy_sigma( + self, + mass: Union[float, npt.NDArray[np.float64]], + z: Union[float, npt.NDArray[np.float64]], + ): + """Return observed scatter corrected by redshift and mass.""" + return MassRichnessGaussian.observed_value( + (self.sigma_p0, self.sigma_p1, self.sigma_p2), + mass, + z, + self.pivot_mass, + self.log1p_pivot_redshift, + ) + + def distribution(self, args, args_map): + return self._distribution_unbinned(args, args_map) diff --git a/firecrown/models/cluster/mass_proxy/gaussian.py b/firecrown/models/cluster/mass_proxy/gaussian.py deleted file mode 100644 index a9cda2430..000000000 --- a/firecrown/models/cluster/mass_proxy/gaussian.py +++ /dev/null @@ -1,61 +0,0 @@ -from typing import List, Union -import numpy.typing as npt -import numpy as np -from scipy import special -from firecrown.models.cluster.kernel import ArgReader, Kernel, KernelType - - -class MassRichnessGaussian(Kernel): - def get_proxy_mean( - self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): - """Return observed quantity corrected by redshift and mass.""" - - def get_proxy_sigma( - self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): - """Return observed scatter corrected by redshift and mass.""" - - def _distribution_binned(self, args: List[float], args_map: ArgReader): - mass = args_map.get_independent_val(args, KernelType.mass) - z = args_map.get_independent_val(args, KernelType.z) - mass_proxy_limits = args_map.get_extra_args(args, self.kernel_type) - - proxy_mean = self.get_proxy_mean(mass, z) - proxy_sigma = self.get_proxy_sigma(mass, z) - - x_min = (proxy_mean - mass_proxy_limits[0] * np.log(10.0)) / ( - np.sqrt(2.0) * proxy_sigma - ) - x_max = (proxy_mean - mass_proxy_limits[1] * np.log(10.0)) / ( - np.sqrt(2.0) * proxy_sigma - ) - - return_vals = np.empty_like(x_min) - mask1 = (x_max > 3.0) | (x_min < -3.0) - mask2 = ~mask1 - - return_vals[mask1] = ( - -(special.erfc(x_min[mask1]) - special.erfc(x_max[mask1])) / 2.0 - ) - return_vals[mask2] = ( - special.erf(x_min[mask2]) - special.erf(x_max[mask2]) - ) / 2.0 - - return return_vals - - def _distribution_unbinned(self, args: List[float], args_map: ArgReader): - mass = args_map.get_independent_val(args, KernelType.mass) - z = args_map.get_independent_val(args, KernelType.z) - mass_proxy = args_map.get_extra_args(args, self.kernel_type) - - proxy_mean = self.get_proxy_mean(mass, z) - proxy_sigma = self.get_proxy_sigma(mass, z) - - return np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( - 2 * np.pi * proxy_sigma - ) diff --git a/firecrown/models/cluster/mass_proxy/murata.py b/firecrown/models/cluster/mass_proxy/murata.py deleted file mode 100644 index fe261fd86..000000000 --- a/firecrown/models/cluster/mass_proxy/murata.py +++ /dev/null @@ -1,86 +0,0 @@ -from typing import List, Tuple, Union, Optional - -import numpy as np -import numpy.typing as npt - -from firecrown import parameters -from firecrown.models.cluster.kernel import KernelType -from firecrown.models.cluster.mass_proxy.gaussian import MassRichnessGaussian - - -def _observed_value( - p: Tuple[float, float, float], - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - pivot_mass: float, - log1p_pivot_redshift: float, -): - """Return observed quantity corrected by redshift and mass.""" - - ln_mass = mass * np.log(10) - delta_ln_mass = ln_mass - pivot_mass - delta_z = np.log1p(z) - log1p_pivot_redshift - - return p[0] + p[1] * delta_ln_mass + p[2] * delta_z - - -class MurataCore(MassRichnessGaussian): - def __init__( - self, - pivot_mass: float, - pivot_redshift: float, - integral_bounds: Optional[List[Tuple[float, float]]] = None, - ): - super().__init__(KernelType.mass_proxy, False, True, integral_bounds) - - self.pivot_redshift = pivot_redshift - self.pivot_mass = pivot_mass * np.log(10.0) # ln(M) - self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) - - # Updatable parameters - self.mu_p0 = parameters.create() - self.mu_p1 = parameters.create() - self.mu_p2 = parameters.create() - self.sigma_p0 = parameters.create() - self.sigma_p1 = parameters.create() - self.sigma_p2 = parameters.create() - - # Verify this gets called last or first - - def get_proxy_mean( - self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): - """Return observed quantity corrected by redshift and mass.""" - return _observed_value( - (self.mu_p0, self.mu_p1, self.mu_p2), - mass, - z, - self.pivot_mass, - self.log1p_pivot_redshift, - ) - - def get_proxy_sigma( - self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): - """Return observed scatter corrected by redshift and mass.""" - return _observed_value( - (self.sigma_p0, self.sigma_p1, self.sigma_p2), - mass, - z, - self.pivot_mass, - self.log1p_pivot_redshift, - ) - - -class MurataBinned(MurataCore): - def distribution(self, args, args_map): - return self._distribution_binned(args, args_map) - - -class MurataUnbinned(MurataCore): - def distribution(self, args, args_map): - return self._distribution_unbinned(args, args_map) diff --git a/tests/test_cluster.py b/tests/test_cluster.py deleted file mode 100644 index abf5f6bcd..000000000 --- a/tests/test_cluster.py +++ /dev/null @@ -1,167 +0,0 @@ -# """Tests for the cluster module.""" - -# from typing import Any, Dict, List -# import itertools -# import math - -# import pytest -# import pyccl as ccl -# import numpy as np - -# from firecrown.models.cluster_mass import ClusterMass, ClusterMassArgument -# from firecrown.models.cluster_redshift import ClusterRedshift, ClusterRedshiftArgument -# from firecrown.models.cluster_mass_true import ClusterMassTrue -# from firecrown.models.cluster_abundance_old import ClusterAbundance -# from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich -# from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec -# from firecrown.parameters import ParamsMap - - -# @pytest.fixture(name="ccl_cosmo") -# def fixture_ccl_cosmo(): -# """Fixture for a CCL cosmology object.""" - -# return ccl.Cosmology( -# Omega_c=0.22, Omega_b=0.0448, h=0.71, sigma8=0.8, n_s=0.963, Neff=3.44 -# ) - - -# @pytest.fixture(name="parameters") -# def fixture_parameters(): -# """Fixture for a parameter map.""" - -# parameters = ParamsMap( -# { -# "mu_p0": 3.19, -# "mu_p1": 0.8, -# "mu_p2": 0.0, -# "sigma_p0": 0.3, -# "sigma_p1": 0.08, -# "sigma_p2": 0.0, -# } -# ) -# return parameters - - -# @pytest.fixture(name="z_args") -# def fixture_cluster_z_args(parameters): -# """Fixture for cluster redshifts.""" -# z_bins = np.array([0.2000146, 0.31251036, 0.42500611, 0.53750187, 0.64999763]) -# cluster_z = ClusterRedshiftSpec() -# assert isinstance(cluster_z, ClusterRedshift) - -# z_args = cluster_z.gen_bins_by_array(z_bins) - -# cluster_z.update(parameters) - -# return z_args - - -# @pytest.fixture(name="logM_args") -# def fixture_cluster_mass_logM_args(parameters): -# """Fixture for cluster masses.""" -# logM_bins = np.array([13.0, 13.5, 14.0, 14.5, 15.0]) -# cluster_mass_t = ClusterMassTrue() -# assert isinstance(cluster_mass_t, ClusterMass) - -# logM_args = cluster_mass_t.gen_bins_by_array(logM_bins) -# cluster_mass_t.update(parameters) - -# return logM_args - - -# @pytest.fixture(name="rich_args") -# def fixture_cluster_mass_rich_args(parameters): -# """Fixture for cluster masses.""" -# pivot_mass = 14.0 -# pivot_redshift = 0.6 -# proxy_bins = np.array([0.45805137, 0.81610273, 1.1741541, 1.53220547, 1.89025684]) - -# cluster_mass_r = ClusterMassRich(pivot_mass, pivot_redshift) -# assert isinstance(cluster_mass_r, ClusterMass) - -# rich_args = cluster_mass_r.gen_bins_by_array(proxy_bins) -# cluster_mass_r.update(parameters) - -# return rich_args - - -# @pytest.fixture(name="cluster_abundance") -# def fixture_cluster_abundance(parameters): -# """Fixture for cluster objects.""" - -# # TODO: remove try/except when pyccl 3.0 is released -# try: -# hmd_200 = ccl.halos.MassDef200c() -# except TypeError: -# hmd_200 = ccl.halos.MassDef200c - -# hmf_args: Dict[str, Any] = {} -# hmf_name = "Bocquet16" -# sky_area = 489 - -# cluster_abundance = ClusterAbundance(hmd_200, hmf_name, hmf_args, sky_area) -# assert isinstance(cluster_abundance, ClusterAbundance) - -# cluster_abundance.update(parameters) - -# return cluster_abundance - - -# def test_initialize_objects( -# ccl_cosmo: ccl.Cosmology, cluster_abundance, z_args, logM_args, rich_args -# ): -# """Test initialization of cluster objects.""" - -# for z_arg in z_args: -# assert isinstance(z_arg, ClusterRedshiftArgument) - -# for logM_arg in logM_args: -# assert isinstance(logM_arg, ClusterMassArgument) - -# for rich_arg in rich_args: -# assert isinstance(rich_arg, ClusterMassArgument) - -# assert isinstance(ccl_cosmo, ccl.Cosmology) -# assert isinstance(cluster_abundance, ClusterAbundance) - - -# def test_cluster_mass_function_compute( -# ccl_cosmo: ccl.Cosmology, -# cluster_abundance: ClusterAbundance, -# z_args: List[ClusterRedshiftArgument], -# logM_args: List[ClusterMassArgument], -# rich_args: List[ClusterMassArgument], -# ): -# """Test cluster mass function computations.""" - -# for redshift_arg, logM_arg, rich_arg in itertools.product( -# z_args, logM_args, rich_args -# ): -# count_rich = cluster_abundance.compute(ccl_cosmo, rich_arg, redshift_arg) -# assert math.isfinite(count_rich) -# assert count_rich > 0.0 -# count_mass = cluster_abundance.compute(ccl_cosmo, logM_arg, redshift_arg) -# assert math.isfinite(count_mass) -# assert count_mass > 0.0 - - -# def test_cluster_mass_function_compute_scipy( -# ccl_cosmo: ccl.Cosmology, -# cluster_abundance: ClusterAbundance, -# z_args: List[ClusterRedshiftArgument], -# logM_args: List[ClusterMassArgument], -# rich_args: List[ClusterMassArgument], -# ): -# """Test cluster mass function computations.""" - -# for redshift_arg, logM_arg, rich_arg in itertools.product( -# z_args, logM_args, rich_args -# ): -# cluster_abundance.prefer_scipy_integration = True -# count_rich = cluster_abundance.compute(ccl_cosmo, rich_arg, redshift_arg) -# assert math.isfinite(count_rich) -# assert count_rich > 0.0 -# count_mass = cluster_abundance.compute(ccl_cosmo, logM_arg, redshift_arg) -# assert math.isfinite(count_mass) -# assert count_mass > 0.0 diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index a0328b767..be61d6b55 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -3,8 +3,9 @@ from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader import pyccl import numpy as np -from typing import List, Tuple, Optional +from typing import List, Tuple, Optional, Any from firecrown.parameters import ParamsMap, create +import math @pytest.fixture() @@ -25,7 +26,7 @@ def __init__( super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) self.param = create() - def distribution(self, args: List[float], args_map: ArgReader): + def distribution(self, args: Tuple[Any, ...], args_map: ArgReader): return 1.0 @@ -129,7 +130,7 @@ def test_abundance_massfunc_accepts_float(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - result = cl_abundance.mass_function(13, 0.1) + result = cl_abundance.mass_function(13.0, 0.1) assert isinstance(result, float) assert result > 0 diff --git a/tests/test_integrator.py b/tests/test_cluster_integrator.py similarity index 97% rename from tests/test_integrator.py rename to tests/test_cluster_integrator.py index 79133accf..dbfbd202c 100644 --- a/tests/test_integrator.py +++ b/tests/test_cluster_integrator.py @@ -1,11 +1,14 @@ import numpy as np import pytest -from typing import List, Tuple -from firecrown.integrator.numcosmo_integrator import ( +from typing import List, Tuple, Optional, Any +from firecrown.models.cluster.integrator.numcosmo_integrator import ( NumCosmoArgReader, NumCosmoIntegrator, ) -from firecrown.integrator.scipy_integrator import ScipyArgReader, ScipyIntegrator +from firecrown.models.cluster.integrator.scipy_integrator import ( + ScipyArgReader, + ScipyIntegrator, +) from firecrown.models.cluster.kernel import KernelType, Kernel, ArgReader from firecrown.models.cluster.abundance import ClusterAbundance @@ -29,11 +32,11 @@ def __init__( kernel_type: KernelType, is_dirac_delta: bool = False, has_analytic_sln: bool = False, - integral_bounds: List[Tuple[float, float]] = None, + integral_bounds: Optional[List[Tuple[float, float]]] = None, ): super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) - def distribution(self, args: List[float], arg_reader: ArgReader): + def distribution(self, args: Tuple[Any, ...], arg_reader: ArgReader): return 1.0 @@ -287,7 +290,7 @@ def test_numcosmo_get_integration_bounds_integrable_kernels( KernelType.mass_proxy.name: 3, } - p_kernel = MockKernel(KernelType.purity, integral_bounds=(0, 1)) + p_kernel = MockKernel(KernelType.purity, integral_bounds=[(0, 1)]) cl_abundance.add_kernel(p_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -360,7 +363,7 @@ def test_numcosmo_get_integration_bounds_analytic_slns( } a_kernel3 = MockKernel( - KernelType.purity, has_analytic_sln=True, integral_bounds=(0, 1) + KernelType.purity, has_analytic_sln=True, integral_bounds=[(0, 1)] ) cl_abundance.add_kernel(a_kernel3) @@ -535,7 +538,7 @@ def test_scipy_get_integration_bounds_integrable_kernels( KernelType.mass_proxy.name: 3, } - p_kernel = MockKernel(KernelType.purity, integral_bounds=(0, 1)) + p_kernel = MockKernel(KernelType.purity, integral_bounds=[(0, 1)]) cl_abundance.add_kernel(p_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -608,7 +611,7 @@ def test_scipy_get_integration_bounds_analytic_slns( } a_kernel3 = MockKernel( - KernelType.purity, has_analytic_sln=True, integral_bounds=(0, 1) + KernelType.purity, has_analytic_sln=True, integral_bounds=[(0, 1)] ) cl_abundance.add_kernel(a_kernel3) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 106f95d01..11cc95cc9 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -74,12 +74,12 @@ def test_create_purity_kernel(): def test_spec_z_distribution(): srk = SpectroscopicRedshift() - assert srk.distribution([0.5], MockArgsReader()) == 1.0 + assert srk.distribution(([0.5], []), MockArgsReader()) == 1.0 def test_true_mass_distribution(): tmk = TrueMass() - assert tmk.distribution([0.5], MockArgsReader()) == 1.0 + assert tmk.distribution(([0.5], []), MockArgsReader()) == 1.0 def test_create_arg_reader(): @@ -112,7 +112,8 @@ def test_purity_distribution(): ] ) - purity = pk.distribution([arguments], map) + purity = pk.distribution((arguments, []), map) + assert isinstance(purity, np.ndarray) for ref, true in zip(purity, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) @@ -141,7 +142,8 @@ def test_completeness_distribution(): ] ) - comp = ck.distribution([bounds], map) + comp = ck.distribution((bounds, []), map) + assert isinstance(comp, np.ndarray) for ref, true in zip(comp, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) @@ -175,6 +177,7 @@ def test_des_photoz_kernel_distribution(): 3.969525474770118, ] - spread = dpk.distribution([bounds], map) + spread = dpk.distribution((bounds, []), map) + assert isinstance(spread, np.ndarray) for ref, true in zip(spread, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 38be8aeba..66450d823 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,9 +1,9 @@ import pytest import numpy as np -from firecrown.models.cluster.mass_proxy.murata import ( +from firecrown.models.cluster.mass_proxy import ( MurataBinned, MurataUnbinned, - _observed_value, + MassRichnessGaussian, ) from firecrown.models.cluster.kernel import ( KernelType, @@ -87,13 +87,13 @@ def test_create_musigma_kernel(): def test_cluster_observed_z(): for z in np.geomspace(1.0e-18, 2.0, 20): - f_z = _observed_value((0.0, 0.0, 1.0), 0.0, z, 0, 0) + f_z = MassRichnessGaussian.observed_value((0.0, 0.0, 1.0), 0.0, z, 0, 0) assert f_z == pytest.approx(np.log1p(z), 1.0e-7, 0.0) def test_cluster_observed_mass(): for logM in np.linspace(10.0, 16.0, 20): - f_logM = _observed_value((0.0, 1.0, 0.0), logM, 0.0, 0, 0) + f_logM = MassRichnessGaussian.observed_value((0.0, 1.0, 0.0), logM, 0.0, 0, 0) assert f_logM == pytest.approx(logM * np.log(10.0), 1.0e-7, 0.0) @@ -140,7 +140,7 @@ def test_cluster_murata_binned_mean(murata_binned_relation: MurataBinned): for z in np.geomspace(1.0e-18, 2.0, 20): test = murata_binned_relation.get_proxy_mean(mass, z) - true = _observed_value( + true = MassRichnessGaussian.observed_value( (3.00, 0.086, 0.01), mass, z, @@ -156,7 +156,7 @@ def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): for z in np.geomspace(1.0e-18, 2.0, 20): test = murata_binned_relation.get_proxy_sigma(mass, z) - true = _observed_value( + true = MassRichnessGaussian.observed_value( (3.00, 0.07, 0.01), mass, z, From 178c57b7e8292296817c049816053c809142a19a Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 31 Oct 2023 10:36:14 -0700 Subject: [PATCH 33/80] Finished refactoring to use an integrand wrapper in each integrator. --- .../cluster_redshift_richness.py | 10 +- .../statistic/binned_cluster_number_counts.py | 10 +- firecrown/models/cluster/abundance.py | 122 +++++++------- .../models/cluster/integrator/integrator.py | 31 ++-- .../cluster/integrator/numcosmo_integrator.py | 153 ++++++++++-------- firecrown/models/cluster/kernel.py | 73 +++++---- firecrown/models/cluster/mass_proxy.py | 62 ++++--- 7 files changed, 270 insertions(+), 191 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index adb1dd2a3..47eafdc41 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -20,7 +20,7 @@ Purity, SpectroscopicRedshift, ) -from firecrown.models.cluster.mass_proxy.murata import MurataBinned +from firecrown.models.cluster.mass_proxy import MurataBinned def get_cluster_abundance(sky_area): @@ -36,15 +36,15 @@ def get_cluster_abundance(sky_area): mass_observable_kernel = MurataBinned(pivot_mass, pivot_redshift) cluster_abundance.add_kernel(mass_observable_kernel) - # redshift_proxy_kernel = SpectroscopicRedshift() - redshift_proxy_kernel = DESY1PhotometricRedshift() + redshift_proxy_kernel = SpectroscopicRedshift() + # redshift_proxy_kernel = DESY1PhotometricRedshift() cluster_abundance.add_kernel(redshift_proxy_kernel) # completeness_kernel = Completeness() # cluster_abundance.add_kernel(completeness_kernel) - purity_kernel = Purity() - cluster_abundance.add_kernel(purity_kernel) + # purity_kernel = Purity() + # cluster_abundance.add_kernel(purity_kernel) return cluster_abundance diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index b253bf06a..fb8f09d73 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -80,12 +80,12 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: if self.use_cluster_counts or self.use_mean_log_mass: for z_proxy_limits, mass_proxy_limits in self.bin_limits: - integrand = tools.cluster_abundance.get_integrand() - bounds, extra_args = self.integrator.get_integration_bounds( + self.integrator.set_integration_bounds( tools.cluster_abundance, z_proxy_limits, mass_proxy_limits ) - counts = self.integrator.integrate(integrand, bounds, extra_args) + integrand = tools.cluster_abundance.get_integrand() + counts = self.integrator.integrate(integrand) cluster_counts.append(counts) theory_vector_list += cluster_counts @@ -94,11 +94,11 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: self.bin_limits, cluster_counts ): integrand = tools.cluster_abundance.get_integrand(avg_mass=True) - bounds, extra_args = self.integrator.get_integration_bounds( + self.integrator.set_integration_bounds( tools.cluster_abundance, z_proxy_limits, mass_proxy_limits ) - total_mass = self.integrator.integrate(integrand, bounds, extra_args) + total_mass = self.integrator.integrate(integrand) mean_mass = total_mass / counts cluster_masses.append(mean_mass) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index a0ccb867f..194946c0f 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,13 +1,14 @@ -from typing import List, Union, Callable, Optional, Dict, Tuple, Any +from typing import List, Callable, Optional, Dict, Tuple from pyccl.cosmology import Cosmology import pyccl.background as bkg import pyccl -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader +from firecrown.models.cluster.kernel import Kernel import numpy as np from firecrown.parameters import ParamsMap import numpy.typing as npt -from functools import singledispatchmethod -import pdb + +# from functools import singledispatchmethod +# import pdb class ClusterAbundance(object): @@ -70,22 +71,22 @@ def update_ingredients( for kernel in self.kernels: kernel.update(params) - @singledispatchmethod - def comoving_volume( - self, z: Union[float, npt.NDArray[np.float64]] - ) -> Union[float, npt.NDArray[np.float64]]: - """Differential Comoving Volume at z. + # @singledispatchmethod + # def comoving_volume( + # self, z: Union[float, npt.NDArray[np.float64]] + # ) -> Union[float, npt.NDArray[np.float64]]: + # """Differential Comoving Volume at z. - parameters - :param ccl_cosmo: pyccl Cosmology - :param z: Cluster Redshift. + # parameters + # :param ccl_cosmo: pyccl Cosmology + # :param z: Cluster Redshift. - :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). - """ - raise ValueError("Unsupported type for z:", type(z)) + # :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). + # """ + # raise ValueError("Unsupported type for z:", type(z)) - @comoving_volume.register(np.ndarray) - def _(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: + # @comoving_volume.register(np.ndarray) + def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) @@ -99,32 +100,32 @@ def _(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: assert isinstance(dV, np.ndarray) return dV * self.sky_area_rad - @comoving_volume.register(float) - def _(self, z: float) -> float: - z_arr = np.atleast_1d(z) - vol = self.comoving_volume(z_arr) - assert isinstance(vol, np.ndarray) - return vol.item() - - @singledispatchmethod + # @comoving_volume.register(float) + # def _(self, z: float) -> float: + # z_arr = np.atleast_1d(z) + # vol = self.comoving_volume(z_arr) + # assert isinstance(vol, np.ndarray) + # return vol.item() + + # @singledispatchmethod + # def mass_function( + # self, + # mass: Union[float, npt.NDArray[np.float64]], + # z: Union[float, npt.NDArray[np.float64]], + # ) -> Union[float, npt.NDArray[np.float64]]: + # """Halo Mass Function at mass and z.""" + # raise ValueError("Unsupported type for either mass or z:", type(mass), type(z)) + + # @mass_function.register(float) + # def _(self, mass: float, z: float) -> float: + # z_arr = np.atleast_1d(z) + # mass_arr = np.atleast_1d(mass) + # mf = self.mass_function(mass_arr, z_arr) + # assert isinstance(mf, np.ndarray) + # return mf.item() + + # @mass_function.register(np.ndarray) def mass_function( - self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ) -> Union[float, npt.NDArray[np.float64]]: - """Halo Mass Function at mass and z.""" - raise ValueError("Unsupported type for either mass or z:", type(mass), type(z)) - - @mass_function.register(float) - def _(self, mass: float, z: float) -> float: - z_arr = np.atleast_1d(z) - mass_arr = np.atleast_1d(mass) - mf = self.mass_function(mass_arr, z_arr) - assert isinstance(mf, np.ndarray) - return mf.item() - - @mass_function.register(np.ndarray) - def _( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] ) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) @@ -141,25 +142,36 @@ def _( def get_integrand( self, avg_mass: bool = False, avg_redshift: bool = False - ) -> Callable[..., npt.NDArray[np.float64]]: - def integrand(*int_args) -> npt.NDArray[np.float64]: - args_map = int_args[-1] - assert isinstance(args_map, ArgReader) - values_for_integral = int_args[:1] - - z = args_map.get_independent_val(values_for_integral, KernelType.z) - mass = args_map.get_independent_val(values_for_integral, KernelType.mass) - + ) -> Callable[ + [ + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + Tuple[float, float], + Tuple[float, float], + ], + npt.NDArray[np.float64], + ]: + def integrand( + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: integrand = self.comoving_volume(z) * self.mass_function(mass, z) + if avg_mass: integrand *= mass if avg_redshift: integrand *= z for kernel in self.kernels: - # Think of overhead here, if its worth it - integrand *= kernel.distribution(int_args, args_map) - return_val = np.atleast_1d(integrand) - return return_val + integrand *= kernel.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) + return integrand return integrand diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 19dda311e..0280b3ec9 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -1,24 +1,33 @@ from abc import ABC, abstractmethod from firecrown.models.cluster.abundance import ClusterAbundance -from typing import Tuple, List, Any -from firecrown.models.cluster.kernel import ArgReader +from typing import Tuple, Callable +import numpy.typing as npt +import numpy as np class Integrator(ABC): - arg_reader: ArgReader - - def __init__(self) -> None: - super().__init__() - @abstractmethod - def integrate(self, integrand, bounds, extra_args): + def integrate( + self, + integrand: Callable[ + [ + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + Tuple[float, float], + Tuple[float, float], + ], + npt.NDArray[np.float64], + ], + ) -> float: """Integrate the integrand over the bounds and include extra_args to integral""" @abstractmethod - def get_integration_bounds( + def set_integration_bounds( self, cl_abundance: ClusterAbundance, z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], - ) -> Tuple[Any, ...]: - """Extract the limits of integration and extra arguments for the integral""" + ) -> None: + """Set the limits of integration and extra arguments for the integral""" diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 52d24fb97..1ecc5ab31 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -1,30 +1,11 @@ from numcosmo_py import Ncm -from typing import Tuple, List, Union, Callable, Any +from typing import Tuple, Callable, Dict import numpy as np import numpy.typing as npt -from firecrown.models.cluster.kernel import KernelType, ArgReader +from firecrown.models.cluster.kernel import KernelType from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.integrator.integrator import Integrator - - -class NumCosmoArgReader(ArgReader): - def __init__(self) -> None: - super().__init__() - self.integral_bounds = dict() - self.extra_args = dict() - - self.integral_bounds_idx = 0 - self.extra_args_idx = 1 - - def get_independent_val( - self, int_args, kernel_type: KernelType - ) -> Union[float, npt.NDArray[np.float64]]: - bounds_values = int_args[self.integral_bounds_idx] - return bounds_values[:, self.integral_bounds[kernel_type.name]] - - def get_extra_args(self, int_args, kernel_type: KernelType): - extra_values = int_args[self.extra_args_idx] - return extra_values[self.extra_args[kernel_type.name]] +import pdb class NumCosmoIntegrator(Integrator): @@ -32,81 +13,117 @@ def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12) -> None: super().__init__() self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance - self.arg_reader = NumCosmoArgReader() - def get_integration_bounds( + self.integral_args_lkp: Dict[KernelType, int] = dict() + self.integral_args_lkp[KernelType.mass] = 0 + self.integral_args_lkp[KernelType.z] = 1 + + self.z_proxy_limits = (-1.0, -1.0) + self.mass_proxy_limits = (-1.0, -1.0) + + def _integral_wrapper( + self, + integrand: Callable[ + [ + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + Tuple[float, float], + Tuple[float, float], + ], + npt.NDArray[np.float64], + ], + ): + def ncm_integrand(int_args): + # pdb.set_trace() + default = np.ones_like(int_args[0]) * -1.0 + + mass = self._get_or_default(int_args, KernelType.mass, default) + z = self._get_or_default(int_args, KernelType.z, default) + mass_proxy = self._get_or_default(int_args, KernelType.mass_proxy, default) + z_proxy = self._get_or_default(int_args, KernelType.z_proxy, default) + + return integrand( + mass, + z, + mass_proxy, + z_proxy, + self.mass_proxy_limits, + self.z_proxy_limits, + ) + + return ncm_integrand + + def set_integration_bounds( self, cl_abundance: ClusterAbundance, z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], - ) -> Tuple[Any, ...]: - self.arg_reader.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - integral_bounds = [ + ) -> None: + # pdb.set_trace() + self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), (cl_abundance.min_z, cl_abundance.max_z), ] - # If any kernel is a dirac delta for z or M, just replace the - # true limits with the proxy limits + self.mass_proxy_limits = mass_proxy_limits + self.z_proxy_limits = z_proxy_limits + for kernel in cl_abundance.dirac_delta_kernels: if kernel.kernel_type == KernelType.z_proxy: - integral_bounds[1] = z_proxy_limits + self.integral_bounds[1] = z_proxy_limits + elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds[0] = mass_proxy_limits + self.integral_bounds[0] = mass_proxy_limits - # If any kernel is not a dirac delta, integrate over the relevant limits - mapping_idx = len(self.arg_reader.integral_bounds.keys()) for kernel in cl_abundance.integrable_kernels: - self.arg_reader.integral_bounds[kernel.kernel_type.name] = mapping_idx - mapping_idx += 1 + idx = len(self.integral_bounds) if kernel.kernel_type == KernelType.z_proxy: - integral_bounds.append(z_proxy_limits) - elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds.append(mass_proxy_limits) - - if kernel.integral_bounds is not None: - integral_bounds.append(*kernel.integral_bounds) - - # Lastly, don't integrate any kernels with an analytic solution - # This means we pass in their limits as extra arguments to the integrator - extra_args = [] - self.arg_reader.extra_args = {} - for i, kernel in enumerate(cl_abundance.analytic_kernels): - self.arg_reader.extra_args[kernel.kernel_type.name] = i + self.integral_bounds.append(z_proxy_limits) + self.integral_args_lkp[KernelType.z_proxy] = idx - if kernel.kernel_type == KernelType.z_proxy: - extra_args.append(z_proxy_limits) elif kernel.kernel_type == KernelType.mass_proxy: - extra_args.append(mass_proxy_limits) - - if kernel.integral_bounds is not None: - extra_args.append(*kernel.integral_bounds) - - return integral_bounds, extra_args + self.integral_bounds.append(mass_proxy_limits) + self.integral_args_lkp[KernelType.mass_proxy] = idx + return - def integrate(self, integrand: Callable, bounds, extra_args): + def integrate( + self, + integrand: Callable[ + [ + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + Tuple[float, float], + Tuple[float, float], + ], + npt.NDArray[np.float64], + ], + ) -> float: Ncm.cfg_init() - int_nd = CountsIntegralND( - len(bounds), - integrand, - extra_args, - self.arg_reader, - ) + ncm_integrand = self._integral_wrapper(integrand) + int_nd = CountsIntegralND(len(self.integral_bounds), ncm_integrand) int_nd.set_method(Ncm.IntegralNDMethod.P_V) int_nd.set_reltol(self._relative_tolerance) int_nd.set_abstol(self._absolute_tolerance) res = Ncm.Vector.new(1) err = Ncm.Vector.new(1) - bl, bu = zip(*bounds) + bl, bu = zip(*self.integral_bounds) int_nd.eval(Ncm.Vector.new_array(bl), Ncm.Vector.new_array(bu), res, err) return res.get(0) + def _get_or_default( + self, int_args, kernel_type, default + ) -> npt.NDArray[np.float64]: + try: + return int_args[:, self.integral_args_lkp[kernel_type]] + except KeyError: + return default + class CountsIntegralND(Ncm.IntegralND): """Integral subclass used by the ClusterAbundance diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 542adb01e..a615c6113 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -54,9 +54,13 @@ def __init__( @abstractmethod def distribution( self, - args: Tuple[Any, ...], - arg_reader: ArgReader, - ) -> Union[float, npt.NDArray[np.float64]]: + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: """The functional form of the distribution or spread of this kernel""" @@ -66,12 +70,13 @@ def __init__(self) -> None: def distribution( self, - args: Tuple[Any, ...], - arg_reader: ArgReader, - ) -> Union[float, npt.NDArray[np.float64]]: - mass = arg_reader.get_independent_val(args, KernelType.mass) - z = arg_reader.get_independent_val(args, KernelType.z) - + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: a_nc = 1.1321 b_nc = 0.7751 a_mc = 13.31 @@ -88,12 +93,13 @@ def __init__(self) -> None: def distribution( self, - args: Tuple[Any, ...], - arg_reader: ArgReader, - ) -> Union[float, npt.NDArray[np.float64]]: - mass_proxy = arg_reader.get_independent_val(args, KernelType.mass_proxy) - z = arg_reader.get_independent_val(args, KernelType.z) - + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: a_nc = np.log(10) * 0.8612 b_nc = np.log(10) * 0.3527 a_rc = 2.2183 @@ -115,10 +121,14 @@ def __init__(self) -> None: def distribution( self, - args: Tuple[Any, ...], - arg_reader: ArgReader, - ) -> Union[float, npt.NDArray[np.float64]]: - return 1.0 + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + return np.atleast_1d(1.0) class SpectroscopicRedshift(Kernel): @@ -127,10 +137,14 @@ def __init__(self) -> None: def distribution( self, - args: Tuple[Any, ...], - arg_reader: ArgReader, - ) -> Union[float, npt.NDArray[np.float64]]: - return 1.0 + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + return np.atleast_1d(1.0) class DESY1PhotometricRedshift(Kernel): @@ -140,12 +154,13 @@ def __init__(self) -> None: def distribution( self, - args: Tuple[Any, ...], - arg_reader: ArgReader, - ) -> Union[float, npt.NDArray[np.float64]]: - z = arg_reader.get_independent_val(args, KernelType.z) - z_proxy = arg_reader.get_independent_val(args, KernelType.z_proxy) - + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index 7959d6733..61479abe7 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -1,11 +1,11 @@ -from typing import List, Tuple, Union, Optional, Any +from typing import List, Tuple, Union, Optional import numpy as np import numpy.typing as npt from firecrown import parameters from scipy import special -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader +from firecrown.models.cluster.kernel import Kernel, KernelType from abc import abstractmethod @@ -42,12 +42,15 @@ def get_proxy_sigma( ) -> Union[float, npt.NDArray[np.float64]]: """Return observed scatter corrected by redshift and mass.""" - def _distribution_binned(self, args: Tuple[Any, ...], args_map: ArgReader): - mass = args_map.get_independent_val(args, KernelType.mass) - z = args_map.get_independent_val(args, KernelType.z) - mass_proxy_limits = args_map.get_extra_args(args, self.kernel_type) - assert isinstance(mass_proxy_limits, tuple) - + def _distribution_binned( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: proxy_mean = self.get_proxy_mean(mass, z) proxy_sigma = self.get_proxy_sigma(mass, z) @@ -71,12 +74,15 @@ def _distribution_binned(self, args: Tuple[Any, ...], args_map: ArgReader): return return_vals - def _distribution_unbinned(self, args: Tuple[Any, ...], args_map: ArgReader): - mass = args_map.get_independent_val(args, KernelType.mass) - z = args_map.get_independent_val(args, KernelType.z) - mass_proxy = args_map.get_extra_args(args, self.kernel_type) - assert isinstance(mass_proxy, float) - + def _distribution_unbinned( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: proxy_mean = self.get_proxy_mean(mass, z) proxy_sigma = self.get_proxy_sigma(mass, z) @@ -136,8 +142,18 @@ def get_proxy_sigma( self.log1p_pivot_redshift, ) - def distribution(self, args, args_map): - return self._distribution_binned(args, args_map) + def distribution( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + return self._distribution_binned( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) class MurataUnbinned(MassRichnessGaussian): @@ -191,5 +207,15 @@ def get_proxy_sigma( self.log1p_pivot_redshift, ) - def distribution(self, args, args_map): - return self._distribution_unbinned(args, args_map) + def distribution( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + return self._distribution_unbinned( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) From 2018c8afaa6d78be90f31d87783c1c84189df297 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 31 Oct 2023 12:43:30 -0700 Subject: [PATCH 34/80] type checking fixes --- .../cluster_redshift_richness.py | 8 +++- .../statistic/binned_cluster_number_counts.py | 2 +- firecrown/models/cluster/abundance_data.py | 4 +- .../cluster/integrator/numcosmo_integrator.py | 37 ++++++++++++------- firecrown/models/cluster/kernel.py | 15 ++++++-- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 47eafdc41..f56f1e86e 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -21,9 +21,11 @@ SpectroscopicRedshift, ) from firecrown.models.cluster.mass_proxy import MurataBinned +from firecrown.likelihood.likelihood import NamedParameters, Likelihood +from typing import Tuple -def get_cluster_abundance(sky_area): +def get_cluster_abundance(sky_area: float) -> ClusterAbundance: hmf = ccl.halos.MassFuncBocquet16() min_mass, max_mass = 13.0, 16.0 min_z, max_z = 0.2, 0.8 @@ -49,7 +51,9 @@ def get_cluster_abundance(sky_area): return cluster_abundance -def build_likelihood(build_parameters): +def build_likelihood( + build_parameters: NamedParameters, +) -> Tuple[Likelihood, ModelingTools]: """ Here we instantiate the number density (or mass function) object. """ diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index fb8f09d73..f670dfca2 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -35,7 +35,7 @@ def __init__( self.integrator = integrator self.data_vector = DataVector.from_list([]) - def read(self, sacc_data: sacc.Sacc): + def read(self, sacc_data: sacc.Sacc) -> None: # Build the data vector and indices needed for the likelihood data_vector = [] diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 148ce9150..c8d5ccaef 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -50,7 +50,9 @@ def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: ) return data_vector_list, sacc_indices_list - def validate_tracers(self, tracers_combinations, data_type: str): + def validate_tracers( + self, tracers_combinations: npt.NDArray, data_type: str + ) -> None: if len(tracers_combinations) == 0: raise ValueError( f"The SACC file does not contain any tracers for the " diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 1ecc5ab31..61638092a 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -1,15 +1,16 @@ from numcosmo_py import Ncm -from typing import Tuple, Callable, Dict +from typing import Tuple, Callable, Dict, Sequence import numpy as np import numpy.typing as npt from firecrown.models.cluster.kernel import KernelType from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.integrator.integrator import Integrator -import pdb class NumCosmoIntegrator(Integrator): - def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12) -> None: + def __init__( + self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 + ) -> None: super().__init__() self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance @@ -18,8 +19,8 @@ def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12) -> None: self.integral_args_lkp[KernelType.mass] = 0 self.integral_args_lkp[KernelType.z] = 1 - self.z_proxy_limits = (-1.0, -1.0) - self.mass_proxy_limits = (-1.0, -1.0) + self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) def _integral_wrapper( self, @@ -34,8 +35,9 @@ def _integral_wrapper( ], npt.NDArray[np.float64], ], - ): - def ncm_integrand(int_args): + ) -> Callable[[npt.NDArray], Sequence[float]]: + # mypy strict issue: npt.NDArray[npt.NDArray[np.float64]] not supported + def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: # pdb.set_trace() default = np.ones_like(int_args[0]) * -1.0 @@ -44,14 +46,16 @@ def ncm_integrand(int_args): mass_proxy = self._get_or_default(int_args, KernelType.mass_proxy, default) z_proxy = self._get_or_default(int_args, KernelType.z_proxy, default) - return integrand( + return_val = integrand( mass, z, mass_proxy, z_proxy, self.mass_proxy_limits, self.z_proxy_limits, - ) + ).tolist() + assert isinstance(return_val, list) + return return_val return ncm_integrand @@ -117,7 +121,11 @@ def integrate( return res.get(0) def _get_or_default( - self, int_args, kernel_type, default + self, + # mypy strict issue: npt.NDArray[npt.NDArray[np.float64]] not supported + int_args: npt.NDArray, + kernel_type: KernelType, + default: npt.NDArray[np.float64], ) -> npt.NDArray[np.float64]: try: return int_args[:, self.integral_args_lkp[kernel_type]] @@ -129,11 +137,14 @@ class CountsIntegralND(Ncm.IntegralND): """Integral subclass used by the ClusterAbundance to compute the integrals using numcosmo.""" - def __init__(self, dim, fun, *args) -> None: + def __init__( + self, + dim: int, + fun: Callable[[npt.NDArray], Sequence[float]], + ) -> None: super().__init__() self.dim = dim self.fun = fun - self.args = args def do_get_dimensions(self) -> Tuple[int, int]: """Get number of dimensions.""" @@ -150,4 +161,4 @@ def do_integrand( ) -> None: """Integrand function.""" x = np.array(x_vec.dup_array()).reshape(npoints, dim) - fval_vec.set_array(self.fun(x, *self.args)) + fval_vec.set_array(self.fun(x)) diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index a615c6113..9740a986c 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -4,6 +4,7 @@ import numpy.typing as npt import numpy as np from firecrown.updatable import Updatable +import pdb class KernelType(Enum): @@ -84,6 +85,7 @@ def distribution( log_mc = a_mc + b_mc * (1.0 + z) nc = a_nc + b_nc * (1.0 + z) completeness = (mass / log_mc) ** nc / ((mass / log_mc) ** nc + 1.0) + assert isinstance(completeness, np.ndarray) return completeness @@ -104,14 +106,19 @@ def distribution( b_nc = np.log(10) * 0.3527 a_rc = 2.2183 b_rc = -0.6592 - - ln_r = np.log(10**mass_proxy) + # pdb.set_trace() + if all(mass_proxy == -1.0): + mean_mass = (mass_proxy_limits[0] + mass_proxy_limits[1]) / 2 + ln_r = np.log(10**mean_mass) + else: + ln_r = np.log(10**mass_proxy) ln_rc = a_rc + b_rc * (1.0 + z) r_over_rc = ln_r / ln_rc nc = a_nc + b_nc * (1.0 + z) purity = (r_over_rc) ** nc / (r_over_rc**nc + 1.0) + assert isinstance(purity, np.ndarray) return purity @@ -164,4 +171,6 @@ def distribution( sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) - return prefactor * distribution + numerator = prefactor * distribution + assert isinstance(numerator, np.ndarray) + return numerator From 8ee7cbf00bd35c2cbf0277a876c6d5a8fb71f4ba Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 31 Oct 2023 15:33:28 -0700 Subject: [PATCH 35/80] Removing ArgReader references, updating tests. --- firecrown/models/cluster/kernel.py | 22 ------- tests/test_cluster_abundance.py | 73 +++++++++++------------ tests/test_cluster_kernels.py | 90 ++++++++++++++++------------- tests/test_cluster_mass_richness.py | 76 +++++++++++------------- 4 files changed, 117 insertions(+), 144 deletions(-) diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 9740a986c..c7c6871ca 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -16,28 +16,6 @@ class KernelType(Enum): purity = 6 -class ArgReader(ABC): - def __init__(self) -> None: - self.integral_bounds: Dict[str, int] = dict() - self.extra_args: Dict[str, int] = dict() - - @abstractmethod - def get_independent_val( - self, - integral_args: Tuple[Any, ...], - kernel_type: KernelType, - ) -> Union[float, npt.NDArray[np.float64]]: - """Returns the current differential value for KernelType""" - - @abstractmethod - def get_extra_args( - self, - integral_args: Tuple[Any, ...], - kernel_type: KernelType, - ) -> Union[float, Tuple[float, float]]: - """Returns the extra arguments passed into the integral for KernelType""" - - class Kernel(Updatable, ABC): def __init__( self, diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index be61d6b55..31f9457f7 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,9 +1,10 @@ import pytest from firecrown.models.cluster.abundance import ClusterAbundance -from firecrown.models.cluster.kernel import Kernel, KernelType, ArgReader +from firecrown.models.cluster.kernel import Kernel, KernelType import pyccl import numpy as np -from typing import List, Tuple, Optional, Any +from typing import List, Tuple, Optional +import numpy.typing as npt from firecrown.parameters import ParamsMap, create import math @@ -26,23 +27,17 @@ def __init__( super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) self.param = create() - def distribution(self, args: Tuple[Any, ...], args_map: ArgReader): - return 1.0 - - -class MockArgsReader(ArgReader): - def __init__(self): - super().__init__() - self.integral_bounds_idx = 0 - self.extra_args_idx = 1 - - def get_independent_val(self, int_args, kernel_type: KernelType): - bounds_values = int_args[self.integral_bounds_idx] - return bounds_values[:, self.integral_bounds[kernel_type.name]] - - def get_extra_args(self, int_args, kernel_type: KernelType): - extra_values = int_args[self.extra_args_idx] - return extra_values[self.extra_args[kernel_type.name]] + def distribution( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + """The functional form of the distribution or spread of this kernel""" + return np.atleast_1d(1.0) def test_cluster_abundance_init(cl_abundance: ClusterAbundance): @@ -106,13 +101,13 @@ def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.integrable_kernels) == 1 -def test_abundance_comoving_vol_accepts_float(cl_abundance: ClusterAbundance): - cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) +# def test_abundance_comoving_vol_accepts_float(cl_abundance: ClusterAbundance): +# cosmo = pyccl.CosmologyVanillaLCDM() +# cl_abundance.update_ingredients(cosmo, ParamsMap()) - result = cl_abundance.comoving_volume(0.1) - assert isinstance(result, float) - assert result > 0 +# result = cl_abundance.comoving_volume(0.1) +# assert isinstance(result, float) +# assert result > 0 def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): @@ -126,13 +121,13 @@ def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): assert np.all(result > 0) -def test_abundance_massfunc_accepts_float(cl_abundance: ClusterAbundance): - cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) +# def test_abundance_massfunc_accepts_float(cl_abundance: ClusterAbundance): +# cosmo = pyccl.CosmologyVanillaLCDM() +# cl_abundance.update_ingredients(cosmo, ParamsMap()) - result = cl_abundance.mass_function(13.0, 0.1) - assert isinstance(result, float) - assert result > 0 +# result = cl_abundance.mass_function(13.0, 0.1) +# assert isinstance(result, float) +# assert result > 0 def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): @@ -151,33 +146,33 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): cl_abundance.update_ingredients(cosmo, ParamsMap()) cl_abundance.add_kernel(MockKernel(KernelType.mass)) - arg_reader = MockArgsReader() - arg_reader.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) - integral_bounds = np.array(list(zip(mass, z))) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) integrand = cl_abundance.get_integrand() assert callable(integrand) - result = integrand(integral_bounds, [], arg_reader) + result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) integrand = cl_abundance.get_integrand(avg_mass=True) assert callable(integrand) - result = integrand(integral_bounds, [], arg_reader) + result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) integrand = cl_abundance.get_integrand(avg_redshift=True) assert callable(integrand) - result = integrand(integral_bounds, [], arg_reader) + result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) integrand = cl_abundance.get_integrand(avg_redshift=True, avg_mass=True) assert callable(integrand) - result = integrand(integral_bounds, [], arg_reader) + result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 11cc95cc9..227acd2b8 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -8,25 +8,9 @@ DESY1PhotometricRedshift, SpectroscopicRedshift, TrueMass, - ArgReader, ) -class MockArgsReader(ArgReader): - def __init__(self): - super().__init__() - self.integral_bounds_idx = 0 - self.extra_args_idx = 1 - - def get_independent_val(self, int_args, kernel_type: KernelType): - bounds_values = int_args[self.integral_bounds_idx] - return bounds_values[:, self.integral_bounds[kernel_type.name]] - - def get_extra_args(self, int_args, kernel_type: KernelType): - extra_values = int_args[self.extra_args_idx] - return extra_values[self.extra_args[kernel_type.name]] - - def test_create_desy1_photometric_redshift_kernel(): drk = DESY1PhotometricRedshift() assert isinstance(drk, Kernel) @@ -74,28 +58,50 @@ def test_create_purity_kernel(): def test_spec_z_distribution(): srk = SpectroscopicRedshift() - assert srk.distribution(([0.5], []), MockArgsReader()) == 1.0 + + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + + assert ( + srk.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) + == 1.0 + ) def test_true_mass_distribution(): tmk = TrueMass() - assert tmk.distribution(([0.5], []), MockArgsReader()) == 1.0 - - -def test_create_arg_reader(): - mr = MockArgsReader() - assert mr.integral_bounds == dict() - assert mr.extra_args == dict() + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + + assert ( + tmk.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) + == 1.0 + ) def test_purity_distribution(): pk = Purity() + + mass = np.linspace(13, 17, 5) mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + z_proxy = np.linspace(0, 1, 5) - arguments = np.array(list(zip(mass_proxy, z))) - map = MockArgsReader() - map.integral_bounds = {KernelType.mass_proxy.name: 0, KernelType.z.name: 1} + mass_proxy_limits = (1.0, 10.0) + z_proxy_limits = (0.1, 1.0) truth = np.array( [ @@ -112,7 +118,9 @@ def test_purity_distribution(): ] ) - purity = pk.distribution((arguments, []), map) + purity = pk.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(purity, np.ndarray) for ref, true in zip(purity, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) @@ -122,10 +130,11 @@ def test_completeness_distribution(): ck = Completeness() mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + z_proxy = np.linspace(0, 1, 5) - bounds = np.array(list(zip(mass, z))) - map = MockArgsReader() - map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} + mass_proxy_limits = (1.0, 10.0) + z_proxy_limits = (0.1, 1.0) truth = np.array( [ @@ -142,7 +151,9 @@ def test_completeness_distribution(): ] ) - comp = ck.distribution((bounds, []), map) + comp = ck.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(comp, np.ndarray) for ref, true in zip(comp, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) @@ -152,17 +163,12 @@ def test_des_photoz_kernel_distribution(): dpk = DESY1PhotometricRedshift() mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) - bounds = np.array(list(zip(mass, z, z_proxy))) - - map = MockArgsReader() - map.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - } + mass_proxy_limits = (1.0, 10.0) + z_proxy_limits = (0.11, 1.01) truth = [ 7.134588921656481, @@ -177,7 +183,9 @@ def test_des_photoz_kernel_distribution(): 3.969525474770118, ] - spread = dpk.distribution((bounds, []), map) + spread = dpk.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(spread, np.ndarray) for ref, true in zip(spread, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 66450d823..4d4f3dc00 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -8,28 +8,12 @@ from firecrown.models.cluster.kernel import ( KernelType, Kernel, - ArgReader, ) PIVOT_Z = 0.6 PIVOT_MASS = 14.625862906 -class MockArgsReader(ArgReader): - def __init__(self): - super().__init__() - self.integral_bounds_idx = 0 - self.extra_args_idx = 1 - - def get_independent_val(self, int_args, kernel_type: KernelType): - bounds_values = int_args[self.integral_bounds_idx] - return bounds_values[:, self.integral_bounds[kernel_type.name]] - - def get_extra_args(self, int_args, kernel_type: KernelType): - extra_values = int_args[self.extra_args_idx] - return extra_values[self.extra_args[kernel_type.name]] - - @pytest.fixture(name="murata_binned_relation") def fixture_murata_binned() -> MurataBinned: """Initialize cluster object.""" @@ -99,21 +83,25 @@ def test_cluster_observed_mass(): def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned): - logM_array = np.linspace(7.0, 26.0, 20) + mass_array = np.linspace(7.0, 26.0, 20) + mass_proxy_limits = (1.0, 5.0) + z_proxy_limits = (0.0, 1.0) + for z in np.geomspace(1.0e-18, 2.0, 20): flip = False - for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): - extra_args = [(1.0, 5.0)] - - args1 = [np.array([[logM_0, z]]), extra_args] - args2 = [np.array([[logM_1, z]]), extra_args] - - args_map = MockArgsReader() - args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - args_map.extra_args = {KernelType.mass_proxy.name: 0} - - probability_0 = murata_binned_relation.distribution(args1, args_map) - probability_1 = murata_binned_relation.distribution(args2, args_map) + for mass1, mass2 in zip(mass_array[:-1], mass_array[1:]): + mass1 = np.atleast_1d(mass1) + mass2 = np.atleast_1d(mass2) + z = np.atleast_1d(z) + z_proxy = np.atleast_1d(0) + mass_proxy = np.atleast_1d(1) + + probability_0 = murata_binned_relation.distribution( + mass1, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) + probability_1 = murata_binned_relation.distribution( + mass2, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert probability_0 >= 0 assert probability_1 >= 0 @@ -168,21 +156,25 @@ def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): def test_cluster_murata_unbinned_distribution(murata_unbinned_relation: MurataUnbinned): - logM_array = np.linspace(7.0, 26.0, 20) + mass_array = np.linspace(7.0, 26.0, 20) + mass_proxy_limits = (1.0, 5.0) + z_proxy_limits = (0.0, 1.0) + for z in np.geomspace(1.0e-18, 2.0, 20): flip = False - for logM_0, logM_1 in zip(logM_array[:-1], logM_array[1:]): - extra_args = [2.5] - - args1 = [np.array([[logM_0, z]]), extra_args] - args2 = [np.array([[logM_1, z]]), extra_args] - - args_map = MockArgsReader() - args_map.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - args_map.extra_args = {KernelType.mass_proxy.name: 0} - - probability_0 = murata_unbinned_relation.distribution(args1, args_map) - probability_1 = murata_unbinned_relation.distribution(args2, args_map) + for mass1, mass2 in zip(mass_array[:-1], mass_array[1:]): + mass1 = np.atleast_1d(mass1) + mass2 = np.atleast_1d(mass2) + z = np.atleast_1d(z) + z_proxy = np.atleast_1d(0) + mass_proxy = np.atleast_1d(1) + + probability_0 = murata_unbinned_relation.distribution( + mass1, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) + probability_1 = murata_unbinned_relation.distribution( + mass2, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) # Probability density should be initially monotonically increasing # and then monotonically decreasing. It should flip only once. From 73b9c63a3c469f5f109177666f23de08cab2f359 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 09:12:13 -0700 Subject: [PATCH 36/80] Implemented scipy integrator! Success! --- firecrown/models/cluster/abundance.py | 24 +- .../models/cluster/integrator/integrator.py | 18 +- .../cluster/integrator/numcosmo_integrator.py | 37 +- .../cluster/integrator/scipy_integrator.py | 137 ++-- tests/test_cluster_integrator.py | 645 ++++-------------- 5 files changed, 221 insertions(+), 640 deletions(-) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 194946c0f..a0380e27c 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -10,6 +10,18 @@ # from functools import singledispatchmethod # import pdb +AbundanceIntegrand = Callable[ + [ + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + npt.NDArray[np.float64], + Tuple[float, float], + Tuple[float, float], + ], + npt.NDArray[np.float64], +] + class ClusterAbundance(object): @property @@ -142,17 +154,7 @@ def mass_function( def get_integrand( self, avg_mass: bool = False, avg_redshift: bool = False - ) -> Callable[ - [ - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - Tuple[float, float], - Tuple[float, float], - ], - npt.NDArray[np.float64], - ]: + ) -> AbundanceIntegrand: def integrand( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 0280b3ec9..833831fec 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -1,25 +1,13 @@ from abc import ABC, abstractmethod -from firecrown.models.cluster.abundance import ClusterAbundance -from typing import Tuple, Callable -import numpy.typing as npt -import numpy as np +from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand +from typing import Tuple class Integrator(ABC): @abstractmethod def integrate( self, - integrand: Callable[ - [ - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - Tuple[float, float], - Tuple[float, float], - ], - npt.NDArray[np.float64], - ], + integrand: AbundanceIntegrand, ) -> float: """Integrate the integrand over the bounds and include extra_args to integral""" diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 61638092a..4d9ff84d1 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -3,7 +3,7 @@ import numpy as np import numpy.typing as npt from firecrown.models.cluster.kernel import KernelType -from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand from firecrown.models.cluster.integrator.integrator import Integrator @@ -15,26 +15,20 @@ def __init__( self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance - self.integral_args_lkp: Dict[KernelType, int] = dict() - self.integral_args_lkp[KernelType.mass] = 0 - self.integral_args_lkp[KernelType.z] = 1 + self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + def _default_integral_args(self) -> Dict[KernelType, int]: + lkp: Dict[KernelType, int] = dict() + lkp[KernelType.mass] = 0 + lkp[KernelType.z] = 1 + return lkp + def _integral_wrapper( self, - integrand: Callable[ - [ - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - Tuple[float, float], - Tuple[float, float], - ], - npt.NDArray[np.float64], - ], + integrand: AbundanceIntegrand, ) -> Callable[[npt.NDArray], Sequence[float]]: # mypy strict issue: npt.NDArray[npt.NDArray[np.float64]] not supported def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: @@ -66,6 +60,7 @@ def set_integration_bounds( mass_proxy_limits: Tuple[float, float], ) -> None: # pdb.set_trace() + self.integral_args_lkp = self._default_integral_args() self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), (cl_abundance.min_z, cl_abundance.max_z), @@ -95,17 +90,7 @@ def set_integration_bounds( def integrate( self, - integrand: Callable[ - [ - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - npt.NDArray[np.float64], - Tuple[float, float], - Tuple[float, float], - ], - npt.NDArray[np.float64], - ], + integrand: AbundanceIntegrand, ) -> float: Ncm.cfg_init() ncm_integrand = self._integral_wrapper(integrand) diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index 4fe00ebbc..ce12063bd 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -1,79 +1,99 @@ from scipy.integrate import nquad -from typing import Tuple, Any +from typing import Callable, Dict, Tuple +import numpy.typing as npt from firecrown.models.cluster.integrator.integrator import Integrator -from firecrown.models.cluster.kernel import KernelType, ArgReader -from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.kernel import KernelType +from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand import numpy as np class ScipyIntegrator(Integrator): - def __init__(self, relative_tolerance=1e-4, absolute_tolerance=1e-12): + def __init__( + self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 + ) -> None: super().__init__() self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance - self.arg_reader = ScipyArgReader() - def get_integration_bounds( + self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() + + self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + + def _default_integral_args(self) -> Dict[KernelType, int]: + lkp: Dict[KernelType, int] = dict() + lkp[KernelType.mass] = 0 + lkp[KernelType.z] = 1 + return lkp + + def _integral_wrapper( + self, + integrand: AbundanceIntegrand, + ) -> Callable[..., float]: + def scipy_integrand(*int_args: float) -> float: + default = -1.0 + + mass = self._get_or_default(int_args, KernelType.mass, default) + z = self._get_or_default(int_args, KernelType.z, default) + mass_proxy = self._get_or_default(int_args, KernelType.mass_proxy, default) + z_proxy = self._get_or_default(int_args, KernelType.z_proxy, default) + + return_val = integrand( + mass, + z, + mass_proxy, + z_proxy, + self.mass_proxy_limits, + self.z_proxy_limits, + ) + + return return_val[0] + + return scipy_integrand + + def set_integration_bounds( self, cl_abundance: ClusterAbundance, z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], - ) -> Tuple[Any, ...]: - self.arg_reader.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - integral_bounds = [ + ) -> None: + # pdb.set_trace() + self.integral_args_lkp = self._default_integral_args() + self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), (cl_abundance.min_z, cl_abundance.max_z), ] - # If any kernel is a dirac delta for z or M, just replace the - # true limits with the proxy limits + self.mass_proxy_limits = mass_proxy_limits + self.z_proxy_limits = z_proxy_limits + for kernel in cl_abundance.dirac_delta_kernels: if kernel.kernel_type == KernelType.z_proxy: - integral_bounds[1] = z_proxy_limits + self.integral_bounds[1] = z_proxy_limits + elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds[0] = mass_proxy_limits + self.integral_bounds[0] = mass_proxy_limits - # If any kernel is not a dirac delta, integrate over the relevant limits - mapping_idx = len(self.arg_reader.integral_bounds.keys()) for kernel in cl_abundance.integrable_kernels: - self.arg_reader.integral_bounds[kernel.kernel_type.name] = mapping_idx - mapping_idx += 1 + idx = len(self.integral_bounds) if kernel.kernel_type == KernelType.z_proxy: - integral_bounds.append(z_proxy_limits) - elif kernel.kernel_type == KernelType.mass_proxy: - integral_bounds.append(mass_proxy_limits) - - if kernel.integral_bounds is not None: - integral_bounds.append(*kernel.integral_bounds) - - # Lastly, don't integrate any kernels with an analytic solution - # This means we pass in their limits as extra arguments to the integrator - extra_args = [] - self.arg_reader.extra_args = {} + self.integral_bounds.append(z_proxy_limits) + self.integral_args_lkp[KernelType.z_proxy] = idx - for kernel in cl_abundance.analytic_kernels: - self.arg_reader.extra_args[kernel.kernel_type.name] = mapping_idx - mapping_idx += 1 - if kernel.kernel_type == KernelType.z_proxy: - extra_args.append(z_proxy_limits) elif kernel.kernel_type == KernelType.mass_proxy: - extra_args.append(mass_proxy_limits) - - if kernel.integral_bounds is not None: - extra_args.append(*kernel.integral_bounds) - - return integral_bounds, extra_args + self.integral_bounds.append(mass_proxy_limits) + self.integral_args_lkp[KernelType.mass_proxy] = idx + return - def integrate(self, integrand, bounds, extra_args): + def integrate( + self, + integrand: AbundanceIntegrand, + ) -> float: + scipy_integrand = self._integral_wrapper(integrand) val = nquad( - integrand, - ranges=bounds, - args=(*extra_args, self.arg_reader), + scipy_integrand, + ranges=self.integral_bounds, opts={ "epsabs": self._absolute_tolerance, "epsrel": self._relative_tolerance, @@ -82,16 +102,13 @@ def integrate(self, integrand, bounds, extra_args): return val - -class ScipyArgReader(ArgReader): - def __init__(self): - super().__init__() - self.integral_bounds = dict() - self.extra_args = dict() - self.integral_bounds_idx = 0 - - def get_independent_val(self, int_args, kernel_type: KernelType): - return np.array(int_args[self.integral_bounds[kernel_type.name]]) - - def get_extra_args(self, int_args, kernel_type: KernelType): - return int_args[self.extra_args[kernel_type.name]] + def _get_or_default( + self, + int_args: Tuple[float, ...], + kernel_type: KernelType, + default: float, + ) -> npt.NDArray[np.float64]: + try: + return np.atleast_1d(int_args[self.integral_args_lkp[kernel_type]]) + except KeyError: + return np.atleast_1d(default) diff --git a/tests/test_cluster_integrator.py b/tests/test_cluster_integrator.py index dbfbd202c..6ad83d433 100644 --- a/tests/test_cluster_integrator.py +++ b/tests/test_cluster_integrator.py @@ -1,15 +1,11 @@ import numpy as np +import numpy.typing as npt import pytest -from typing import List, Tuple, Optional, Any +from typing import List, Tuple, Optional from firecrown.models.cluster.integrator.numcosmo_integrator import ( - NumCosmoArgReader, NumCosmoIntegrator, ) -from firecrown.models.cluster.integrator.scipy_integrator import ( - ScipyArgReader, - ScipyIntegrator, -) -from firecrown.models.cluster.kernel import KernelType, Kernel, ArgReader +from firecrown.models.cluster.kernel import KernelType, Kernel from firecrown.models.cluster.abundance import ClusterAbundance @@ -36,108 +32,17 @@ def __init__( ): super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) - def distribution(self, args: Tuple[Any, ...], arg_reader: ArgReader): - return 1.0 - - -def test_numcosmo_argreader_extra_args(): - arg_reader = NumCosmoArgReader() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - arg_reader.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - - extra_args = [(1, 2, 3), ("hello world")] - arg_reader.extra_args = {KernelType.mass_proxy.name: 0, KernelType.z_proxy.name: 1} - - integral_bounds = np.array(list(zip(mass, z))) - int_args = [integral_bounds, extra_args] - - assert arg_reader.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) - assert arg_reader.get_extra_args(int_args, KernelType.z_proxy) == "hello world" - - -def test_numcosmo_argreader_integral_bounds(): - arg_reader = NumCosmoArgReader() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) - mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) - - arg_reader.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - } - - integral_bounds = np.array(list(zip(mass, z, z_proxy, mass_proxy))) - int_args = [integral_bounds] - - assert (mass == arg_reader.get_independent_val(int_args, KernelType.mass)).all() - assert (z == arg_reader.get_independent_val(int_args, KernelType.z)).all() - assert ( - z_proxy == arg_reader.get_independent_val(int_args, KernelType.z_proxy) - ).all() - assert ( - mass_proxy == arg_reader.get_independent_val(int_args, KernelType.mass_proxy) - ).all() - - -def test_create_numcosmo_argreader(): - am = NumCosmoArgReader() - assert am.integral_bounds == dict() - assert am.extra_args == dict() - assert am.integral_bounds_idx == 0 - assert am.extra_args_idx == 1 - - -def test_scipy_argreader_extra_args(): - arg_reader = ScipyArgReader() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - arg_reader.integral_bounds = {KernelType.mass.name: 0, KernelType.z.name: 1} - - extra_args = [(1, 2, 3), ("hello world")] - arg_reader.extra_args = {KernelType.mass_proxy.name: 2, KernelType.z_proxy.name: 3} - - for m_i, z_i in list(zip(mass, z)): - int_args = [m_i, z_i, *extra_args] - - assert arg_reader.get_extra_args(int_args, KernelType.mass_proxy) == (1, 2, 3) - assert arg_reader.get_extra_args(int_args, KernelType.z_proxy) == "hello world" - - -def test_scipy_argreader_integral_bounds(): - arg_reader = ScipyArgReader() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) - mass_proxy = np.array([0.9, 1.9, 2.9, 3.9, 4.9, 5.9, 6.9, 7.9, 8.9, 9.9]) - - arg_reader.integral_bounds = { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - } - - for m_i, z_i, zp_i, mp_i in list(zip(mass, z, z_proxy, mass_proxy)): - int_args = [m_i, z_i, zp_i, mp_i] - assert m_i == arg_reader.get_independent_val(int_args, KernelType.mass) - assert z_i == arg_reader.get_independent_val(int_args, KernelType.z) - assert zp_i == arg_reader.get_independent_val(int_args, KernelType.z_proxy) - assert mp_i == arg_reader.get_independent_val(int_args, KernelType.mass_proxy) - - -def test_create_scipy_argreader(): - am = ScipyArgReader() - assert am.integral_bounds == dict() - assert am.extra_args == dict() - assert am.integral_bounds_idx == 0 + def distribution( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + """The functional form of the distribution or spread of this kernel""" + return np.atleast_1d(1.0) def test_numcosmo_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): @@ -149,17 +54,15 @@ def test_numcosmo_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundan m_bins = list(zip(m_array[:-1], m_array[1:])) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 2 + assert nci.integral_bounds == [(13, 17), (0, 2)] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, } @@ -175,17 +78,15 @@ def test_numcosmo_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [mass_limits, (0, 2)] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 2 + assert nci.integral_bounds == [mass_limits, (0, 2)] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, } cl_abundance.kernels.clear() @@ -193,34 +94,29 @@ def test_numcosmo_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [(13, 17), z_limits] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 2 + assert nci.integral_bounds == [(13, 17), z_limits] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [mass_limits, z_limits] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 2 + assert nci.integral_bounds == [mass_limits, z_limits] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, } @@ -238,18 +134,16 @@ def test_numcosmo_get_integration_bounds_integrable_kernels( cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 3 - assert bounds == [(13, 17), (0, 2), mass_limits] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.mass_proxy.name: 2, + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 3 + assert nci.integral_bounds == [(13, 17), (0, 2), mass_limits] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + KernelType.mass_proxy: 2, } cl_abundance.kernels.clear() @@ -257,57 +151,33 @@ def test_numcosmo_get_integration_bounds_integrable_kernels( cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 3 - assert bounds == [(13, 17), (0, 2), z_limits] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 3 + assert nci.integral_bounds == [(13, 17), (0, 2), z_limits] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + KernelType.z_proxy: 2, } ig_kernel2 = MockKernel(KernelType.mass_proxy) cl_abundance.add_kernel(ig_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 4 - assert bounds == [(13, 17), (0, 2), z_limits, mass_limits] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - } + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - p_kernel = MockKernel(KernelType.purity, integral_bounds=[(0, 1)]) - cl_abundance.add_kernel(p_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 5 - assert bounds == [(13, 17), (0, 2), z_limits, mass_limits, (0, 1)] - assert len(nci.arg_reader.extra_args) == 0 - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - KernelType.purity.name: 4, + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 4 + assert nci.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + KernelType.z_proxy: 2, + KernelType.mass_proxy: 3, } @@ -325,336 +195,55 @@ def test_numcosmo_get_integration_bounds_analytic_slns( cl_abundance.add_kernel(a_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 1 - assert extra_args == [mass_limits] - assert nci.arg_reader.extra_args == {KernelType.mass_proxy.name: 0} - - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) - cl_abundance.add_kernel(a_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 2 - assert extra_args == [mass_limits, z_limits] - assert nci.arg_reader.extra_args == { - KernelType.mass_proxy.name: 0, - KernelType.z_proxy.name: 1, - } - - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - a_kernel3 = MockKernel( - KernelType.purity, has_analytic_sln=True, integral_bounds=[(0, 1)] - ) - cl_abundance.add_kernel(a_kernel3) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = nci.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 3 - assert extra_args == [mass_limits, z_limits, (0, 1)] - assert nci.arg_reader.extra_args == { - KernelType.mass_proxy.name: 0, - KernelType.z_proxy.name: 1, - KernelType.purity.name: 2, - } - - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert nci.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - -def test_scipy_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - -def test_scipy_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - dd_kernel = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) - cl_abundance.add_kernel(dd_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [mass_limits, (0, 2)] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - cl_abundance.kernels.clear() - dd_kernel = MockKernel(KernelType.z_proxy, is_dirac_delta=True) - cl_abundance.add_kernel(dd_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [(13, 17), z_limits] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) - cl_abundance.add_kernel(dd_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 2 - assert bounds == [mass_limits, z_limits] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - -def test_scipy_get_integration_bounds_integrable_kernels( - cl_abundance: ClusterAbundance, -): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - ig_kernel = MockKernel(KernelType.mass_proxy) - cl_abundance.add_kernel(ig_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 3 - assert bounds == [(13, 17), (0, 2), mass_limits] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.mass_proxy.name: 2, - } - - cl_abundance.kernels.clear() - ig_kernel = MockKernel(KernelType.z_proxy) - cl_abundance.add_kernel(ig_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 3 - assert bounds == [(13, 17), (0, 2), z_limits] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - } - - ig_kernel2 = MockKernel(KernelType.mass_proxy) - cl_abundance.add_kernel(ig_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 4 - assert bounds == [(13, 17), (0, 2), z_limits, mass_limits] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - } - - p_kernel = MockKernel(KernelType.purity, integral_bounds=[(0, 1)]) - cl_abundance.add_kernel(p_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 0 - assert len(bounds) == 5 - assert bounds == [(13, 17), (0, 2), z_limits, mass_limits, (0, 1)] - assert len(spi.arg_reader.extra_args) == 0 - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - KernelType.z_proxy.name: 2, - KernelType.mass_proxy.name: 3, - KernelType.purity.name: 4, - } - - -def test_scipy_get_integration_bounds_analytic_slns( - cl_abundance: ClusterAbundance, -): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - a_kernel = MockKernel(KernelType.mass_proxy, has_analytic_sln=True) - cl_abundance.add_kernel(a_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 1 - assert extra_args == [mass_limits] - assert spi.arg_reader.extra_args == {KernelType.mass_proxy.name: 2} - - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 2 + assert nci.integral_bounds == [(13, 17), (0, 2)] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, } a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) cl_abundance.add_kernel(a_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 2 - assert extra_args == [mass_limits, z_limits] - assert spi.arg_reader.extra_args == { - KernelType.mass_proxy.name: 2, - KernelType.z_proxy.name: 3, - } - - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, - } - - a_kernel3 = MockKernel( - KernelType.purity, has_analytic_sln=True, integral_bounds=[(0, 1)] - ) - cl_abundance.add_kernel(a_kernel3) - - for z_limits, mass_limits in zip(z_bins, m_bins): - bounds, extra_args = spi.get_integration_bounds( - cl_abundance, z_limits, mass_limits - ) - - assert len(extra_args) == 3 - assert extra_args == [mass_limits, z_limits, (0, 1)] - assert spi.arg_reader.extra_args == { - KernelType.mass_proxy.name: 2, - KernelType.z_proxy.name: 3, - KernelType.purity.name: 4, - } + nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - assert len(bounds) == 2 - assert bounds == [(13, 17), (0, 2)] - assert spi.arg_reader.integral_bounds == { - KernelType.mass.name: 0, - KernelType.z.name: 1, + assert nci.mass_proxy_limits == mass_limits + assert nci.z_proxy_limits == z_limits + assert len(nci.integral_bounds) == 2 + assert nci.integral_bounds == [(13, 17), (0, 2)] + assert nci.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, } -def test_scipy_integrator_integrate(): - sci = ScipyIntegrator() - - def integrand(*int_args): - x = int_args[0] - return x - - bounds = [(0, 1)] - result = sci.integrate(integrand, bounds, []) - assert result == 0.5 - - -def test_numcosmo_integrator_integrate(): +def test_numcosmo_integrator_integrate(cl_abundance: ClusterAbundance): nci = NumCosmoIntegrator() - - def integrand(*int_args): - x = int_args[0] + cl_abundance.min_mass = 0 + cl_abundance.max_mass = 1 + cl_abundance.min_z = 0 + cl_abundance.max_z = 1 + + def integrand( + a: npt.NDArray[np.float64], + b: npt.NDArray[np.float64], + c: npt.NDArray[np.float64], + d: npt.NDArray[np.float64], + e: Tuple[float, float], + f: Tuple[float, float], + ): + x = a[0] return x - bounds = [(0, 1)] - result = nci.integrate(integrand, bounds, []) + nci.set_integration_bounds( + cl_abundance, + (0, 1), + (0, 1), + ) + result = nci.integrate(integrand) assert result == 0.5 From caa1e3e74ab1b1864bca78ee018040c54ce01ff1 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 09:41:21 -0700 Subject: [PATCH 37/80] Added test coverage for scipy All cluster modules at 100% test coverage. --- firecrown/models/cluster/abundance.py | 42 --- .../cluster/integrator/numcosmo_integrator.py | 1 - tests/test_cluster_abundance.py | 65 +++-- ...py => test_cluster_integrator_numcosmo.py} | 16 +- tests/test_cluster_integrator_scipy.py | 251 ++++++++++++++++++ tests/test_cluster_kernels.py | 41 ++- 6 files changed, 345 insertions(+), 71 deletions(-) rename tests/{test_cluster_integrator.py => test_cluster_integrator_numcosmo.py} (95%) create mode 100644 tests/test_cluster_integrator_scipy.py diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index a0380e27c..217a24c81 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -7,8 +7,6 @@ from firecrown.parameters import ParamsMap import numpy.typing as npt -# from functools import singledispatchmethod -# import pdb AbundanceIntegrand = Callable[ [ @@ -83,21 +81,6 @@ def update_ingredients( for kernel in self.kernels: kernel.update(params) - # @singledispatchmethod - # def comoving_volume( - # self, z: Union[float, npt.NDArray[np.float64]] - # ) -> Union[float, npt.NDArray[np.float64]]: - # """Differential Comoving Volume at z. - - # parameters - # :param ccl_cosmo: pyccl Cosmology - # :param z: Cluster Redshift. - - # :return: Differential Comoving Volume at z in units of Mpc^3 (comoving). - # """ - # raise ValueError("Unsupported type for z:", type(z)) - - # @comoving_volume.register(np.ndarray) def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) @@ -112,31 +95,6 @@ def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64] assert isinstance(dV, np.ndarray) return dV * self.sky_area_rad - # @comoving_volume.register(float) - # def _(self, z: float) -> float: - # z_arr = np.atleast_1d(z) - # vol = self.comoving_volume(z_arr) - # assert isinstance(vol, np.ndarray) - # return vol.item() - - # @singledispatchmethod - # def mass_function( - # self, - # mass: Union[float, npt.NDArray[np.float64]], - # z: Union[float, npt.NDArray[np.float64]], - # ) -> Union[float, npt.NDArray[np.float64]]: - # """Halo Mass Function at mass and z.""" - # raise ValueError("Unsupported type for either mass or z:", type(mass), type(z)) - - # @mass_function.register(float) - # def _(self, mass: float, z: float) -> float: - # z_arr = np.atleast_1d(z) - # mass_arr = np.atleast_1d(mass) - # mf = self.mass_function(mass_arr, z_arr) - # assert isinstance(mf, np.ndarray) - # return mf.item() - - # @mass_function.register(np.ndarray) def mass_function( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] ) -> npt.NDArray[np.float64]: diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 4d9ff84d1..020a4d425 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -32,7 +32,6 @@ def _integral_wrapper( ) -> Callable[[npt.NDArray], Sequence[float]]: # mypy strict issue: npt.NDArray[npt.NDArray[np.float64]] not supported def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: - # pdb.set_trace() default = np.ones_like(int_args[0]) * -1.0 mass = self._get_or_default(int_args, KernelType.mass, default) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 31f9457f7..211b16b6c 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -59,6 +59,7 @@ def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): assert cl_abundance.cosmo is None assert mk.param is None + assert cl_abundance._hmf_cache == {} pmap = ParamsMap({"param": 42}) cosmo = pyccl.CosmologyVanillaLCDM() @@ -67,6 +68,13 @@ def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): assert cl_abundance.cosmo is not None assert cl_abundance.cosmo == cosmo assert mk.param == 42 + assert cl_abundance._hmf_cache == {} + + cl_abundance.update_ingredients(None) + assert cl_abundance.cosmo is None + cl_abundance.update_ingredients(cosmo) + assert cl_abundance.cosmo is not None + assert cl_abundance.cosmo == cosmo def test_cluster_sky_area(cl_abundance: ClusterAbundance): @@ -101,15 +109,6 @@ def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.integrable_kernels) == 1 -# def test_abundance_comoving_vol_accepts_float(cl_abundance: ClusterAbundance): -# cosmo = pyccl.CosmologyVanillaLCDM() -# cl_abundance.update_ingredients(cosmo, ParamsMap()) - -# result = cl_abundance.comoving_volume(0.1) -# assert isinstance(result, float) -# assert result > 0 - - def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -121,15 +120,6 @@ def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): assert np.all(result > 0) -# def test_abundance_massfunc_accepts_float(cl_abundance: ClusterAbundance): -# cosmo = pyccl.CosmologyVanillaLCDM() -# cl_abundance.update_ingredients(cosmo, ParamsMap()) - -# result = cl_abundance.mass_function(13.0, 0.1) -# assert isinstance(result, float) -# assert result > 0 - - def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -159,18 +149,57 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) + +def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.add_kernel(MockKernel(KernelType.mass)) + + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + integrand = cl_abundance.get_integrand(avg_mass=True) assert callable(integrand) result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) + +def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.add_kernel(MockKernel(KernelType.mass)) + + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + integrand = cl_abundance.get_integrand(avg_redshift=True) assert callable(integrand) result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) + +def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.add_kernel(MockKernel(KernelType.mass)) + + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + integrand = cl_abundance.get_integrand(avg_redshift=True, avg_mass=True) assert callable(integrand) result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) diff --git a/tests/test_cluster_integrator.py b/tests/test_cluster_integrator_numcosmo.py similarity index 95% rename from tests/test_cluster_integrator.py rename to tests/test_cluster_integrator_numcosmo.py index 6ad83d433..0ddead708 100644 --- a/tests/test_cluster_integrator.py +++ b/tests/test_cluster_integrator_numcosmo.py @@ -45,7 +45,7 @@ def distribution( return np.atleast_1d(1.0) -def test_numcosmo_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): +def test_numcosmo_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): nci = NumCosmoIntegrator() z_array = np.linspace(0, 2, 10) @@ -66,7 +66,7 @@ def test_numcosmo_get_integration_bounds_no_kernels(cl_abundance: ClusterAbundan } -def test_numcosmo_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): +def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): nci = NumCosmoIntegrator() z_array = np.linspace(0, 2, 10) @@ -120,7 +120,7 @@ def test_numcosmo_get_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda } -def test_numcosmo_get_integration_bounds_integrable_kernels( +def test_numcosmo_set_integration_bounds_integrable_kernels( cl_abundance: ClusterAbundance, ): nci = NumCosmoIntegrator() @@ -181,7 +181,7 @@ def test_numcosmo_get_integration_bounds_integrable_kernels( } -def test_numcosmo_get_integration_bounds_analytic_slns( +def test_numcosmo_set_integration_bounds_analytic_slns( cl_abundance: ClusterAbundance, ): nci = NumCosmoIntegrator() @@ -237,8 +237,9 @@ def integrand( e: Tuple[float, float], f: Tuple[float, float], ): - x = a[0] - return x + # xy + result = a * b + return result nci.set_integration_bounds( cl_abundance, @@ -246,4 +247,5 @@ def integrand( (0, 1), ) result = nci.integrate(integrand) - assert result == 0.5 + # \int_0^1 \int_0^1 xy dx dy = 1/4 + assert result == 0.25 diff --git a/tests/test_cluster_integrator_scipy.py b/tests/test_cluster_integrator_scipy.py new file mode 100644 index 000000000..7a6b67c4e --- /dev/null +++ b/tests/test_cluster_integrator_scipy.py @@ -0,0 +1,251 @@ +import numpy as np +import numpy.typing as npt +import pytest +from typing import List, Tuple, Optional +from firecrown.models.cluster.integrator.scipy_integrator import ( + ScipyIntegrator, +) +from firecrown.models.cluster.kernel import KernelType, Kernel +from firecrown.models.cluster.abundance import ClusterAbundance + + +@pytest.fixture(name="cl_abundance") +def fixture_cl_abundance(): + cl_abundance = ClusterAbundance( + min_z=0, + max_z=2, + min_mass=13, + max_mass=17, + sky_area=100, + halo_mass_function=None, + ) + return cl_abundance + + +class MockKernel(Kernel): + def __init__( + self, + kernel_type: KernelType, + is_dirac_delta: bool = False, + has_analytic_sln: bool = False, + integral_bounds: Optional[List[Tuple[float, float]]] = None, + ): + super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) + + def distribution( + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + mass_proxy: npt.NDArray[np.float64], + z_proxy: npt.NDArray[np.float64], + mass_proxy_limits: Tuple[float, float], + z_proxy_limits: Tuple[float, float], + ) -> npt.NDArray[np.float64]: + """The functional form of the distribution or spread of this kernel""" + return np.atleast_1d(1.0) + + +def test_scipy_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 2 + assert spi.integral_bounds == [(13, 17), (0, 2)] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } + + +def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + dd_kernel = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 2 + assert spi.integral_bounds == [mass_limits, (0, 2)] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } + + cl_abundance.kernels.clear() + dd_kernel = MockKernel(KernelType.z_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 2 + assert spi.integral_bounds == [(13, 17), z_limits] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } + dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + cl_abundance.add_kernel(dd_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 2 + assert spi.integral_bounds == [mass_limits, z_limits] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } + + +def test_scipy_set_integration_bounds_integrable_kernels( + cl_abundance: ClusterAbundance, +): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + ig_kernel = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 3 + assert spi.integral_bounds == [(13, 17), (0, 2), mass_limits] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + KernelType.mass_proxy: 2, + } + + cl_abundance.kernels.clear() + ig_kernel = MockKernel(KernelType.z_proxy) + cl_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 3 + assert spi.integral_bounds == [(13, 17), (0, 2), z_limits] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + KernelType.z_proxy: 2, + } + + ig_kernel2 = MockKernel(KernelType.mass_proxy) + cl_abundance.add_kernel(ig_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 4 + assert spi.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + KernelType.z_proxy: 2, + KernelType.mass_proxy: 3, + } + + +def test_scipy_set_integration_bounds_analytic_slns( + cl_abundance: ClusterAbundance, +): + spi = ScipyIntegrator() + + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + a_kernel = MockKernel(KernelType.mass_proxy, has_analytic_sln=True) + cl_abundance.add_kernel(a_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 2 + assert spi.integral_bounds == [(13, 17), (0, 2)] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } + + a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) + cl_abundance.add_kernel(a_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) + + assert spi.mass_proxy_limits == mass_limits + assert spi.z_proxy_limits == z_limits + assert len(spi.integral_bounds) == 2 + assert spi.integral_bounds == [(13, 17), (0, 2)] + assert spi.integral_args_lkp == { + KernelType.mass: 0, + KernelType.z: 1, + } + + +def test_scipy_integrator_integrate(cl_abundance: ClusterAbundance): + spi = ScipyIntegrator() + cl_abundance.min_mass = 0 + cl_abundance.max_mass = 1 + cl_abundance.min_z = 0 + cl_abundance.max_z = 1 + + def integrand( + a: npt.NDArray[np.float64], + b: npt.NDArray[np.float64], + c: npt.NDArray[np.float64], + d: npt.NDArray[np.float64], + e: Tuple[float, float], + f: Tuple[float, float], + ): + # xy + result = a * b + return result + + spi.set_integration_bounds( + cl_abundance, + (0, 1), + (0, 1), + ) + result = spi.integrate(integrand) + # \int_0^1 \int_0^1 xy dx dy = 1/4 + assert result == pytest.approx(0.25, rel=1e-15, abs=0) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 227acd2b8..d5656ee88 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -94,11 +94,11 @@ def test_true_mass_distribution(): def test_purity_distribution(): pk = Purity() - mass = np.linspace(13, 17, 5) + mass = np.linspace(13, 17, 10) mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - z_proxy = np.linspace(0, 1, 5) + z_proxy = np.linspace(0, 1, 10) mass_proxy_limits = (1.0, 10.0) z_proxy_limits = (0.1, 1.0) @@ -126,12 +126,47 @@ def test_purity_distribution(): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) +def test_purity_distribution_uses_mean(): + pk = Purity() + + mass = np.linspace(13, 17, 10) + z_proxy = np.linspace(0, 1, 10) + + z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) + mass_proxy = np.ones_like(z) * -1.0 + + mass_proxy_limits = (1.0, 10.0) + z_proxy_limits = (0.1, 1.0) + + truth = np.array( + [ + 0.9978693724040568, + 0.9984319673134954, + 0.9988620014089232, + 0.9991864843696077, + 0.9994279315032029, + 0.999604893383804, + 0.9997324678841709, + 0.9998227843987537, + 0.9998854531462606, + 0.9999279749997235, + ] + ) + + purity = pk.distribution( + mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) + assert isinstance(purity, np.ndarray) + for ref, true in zip(purity, truth): + assert ref == pytest.approx(true, rel=1e-7, abs=0.0) + + def test_completeness_distribution(): ck = Completeness() mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z_proxy = np.linspace(0, 1, 5) + z_proxy = np.linspace(0, 1, 10) mass_proxy_limits = (1.0, 10.0) z_proxy_limits = (0.1, 1.0) From e8cd9dc1b89edfe4658cb7228185e52d5886293f Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 09:53:23 -0700 Subject: [PATCH 38/80] Fixing all mypy strict errors except generic type npt.NDarray (no way around this) Updating mass proxy tests. --- .../statistic/binned_cluster_number_counts.py | 3 - .../cluster/integrator/scipy_integrator.py | 8 +-- firecrown/models/cluster/mass_proxy.py | 54 ++++++++-------- .../statistic/test_cluster_number_counts.py | 61 ------------------- tests/test_cluster_mass_richness.py | 16 +++-- 5 files changed, 45 insertions(+), 97 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index f670dfca2..53b7359df 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -13,8 +13,6 @@ from firecrown.modeling_tools import ModelingTools import numpy as np -import cProfile - class BinnedClusterNumberCounts(Statistic): def __init__( @@ -26,7 +24,6 @@ def __init__( systematics: Optional[List[SourceSystematic]] = None, ): super().__init__() - self.pr = cProfile.Profile() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None self.use_cluster_counts = cluster_counts diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index ce12063bd..d322a2b8f 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -45,9 +45,9 @@ def scipy_integrand(*int_args: float) -> float: z_proxy, self.mass_proxy_limits, self.z_proxy_limits, - ) - - return return_val[0] + )[0] + assert isinstance(return_val, float) + return return_val return scipy_integrand @@ -99,7 +99,7 @@ def integrate( "epsrel": self._relative_tolerance, }, )[0] - + assert isinstance(val, float) return val def _get_or_default( diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index 61479abe7..c0f3a103e 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Union, Optional +from typing import List, Tuple, Optional import numpy as np import numpy.typing as npt @@ -13,33 +13,35 @@ class MassRichnessGaussian(Kernel): @staticmethod def observed_value( p: Tuple[float, float, float], - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], pivot_mass: float, log1p_pivot_redshift: float, - ) -> Union[float, npt.NDArray[np.float64]]: + ) -> npt.NDArray[np.float64]: """Return observed quantity corrected by redshift and mass.""" ln_mass = mass * np.log(10) delta_ln_mass = ln_mass - pivot_mass delta_z = np.log1p(z) - log1p_pivot_redshift - return p[0] + p[1] * delta_ln_mass + p[2] * delta_z + result = p[0] + p[1] * delta_ln_mass + p[2] * delta_z + assert isinstance(result, np.ndarray) + return result @abstractmethod def get_proxy_mean( self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ) -> Union[float, npt.NDArray[np.float64]]: + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + ) -> npt.NDArray[np.float64]: """Return observed quantity corrected by redshift and mass.""" @abstractmethod def get_proxy_sigma( self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ) -> Union[float, npt.NDArray[np.float64]]: + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + ) -> npt.NDArray[np.float64]: """Return observed scatter corrected by redshift and mass.""" def _distribution_binned( @@ -71,7 +73,7 @@ def _distribution_binned( return_vals[mask2] = ( special.erf(x_min[mask2]) - special.erf(x_max[mask2]) ) / 2.0 - + assert isinstance(return_vals, np.ndarray) return return_vals def _distribution_unbinned( @@ -86,9 +88,11 @@ def _distribution_unbinned( proxy_mean = self.get_proxy_mean(mass, z) proxy_sigma = self.get_proxy_sigma(mass, z) - return np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( + result = np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( 2 * np.pi * proxy_sigma ) + assert isinstance(result, np.ndarray) + return result class MurataBinned(MassRichnessGaussian): @@ -116,9 +120,9 @@ def __init__( def get_proxy_mean( self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + ) -> npt.NDArray[np.float64]: """Return observed quantity corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), @@ -130,9 +134,9 @@ def get_proxy_mean( def get_proxy_sigma( self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + ) -> npt.NDArray[np.float64]: """Return observed scatter corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.sigma_p0, self.sigma_p1, self.sigma_p2), @@ -181,9 +185,9 @@ def __init__( def get_proxy_mean( self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + ) -> npt.NDArray[np.float64]: """Return observed quantity corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), @@ -195,9 +199,9 @@ def get_proxy_mean( def get_proxy_sigma( self, - mass: Union[float, npt.NDArray[np.float64]], - z: Union[float, npt.NDArray[np.float64]], - ): + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], + ) -> npt.NDArray[np.float64]: """Return observed scatter corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.sigma_p0, self.sigma_p1, self.sigma_p2), diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index c4a91ddfb..e69de29bb 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -1,61 +0,0 @@ -# """Tests for the ClusterNumberCounts statistic. -# """ -# import pytest - -# import pyccl.halos -# import sacc - -# from firecrown.likelihood.gauss_family.statistic.cluster_number_counts import ( -# ClusterNumberCounts, -# ClusterAbundance, -# ) -# from firecrown.models.cluster_mass_rich_proxy import ClusterMassRich -# from firecrown.models.cluster_redshift_spec import ClusterRedshiftSpec - - -# @pytest.fixture(name="minimal_stat") -# def fixture_minimal_stat() -> ClusterNumberCounts: -# """Return a correctly initialized :python:`ClusterNumberCounts` object.""" -# stat = ClusterNumberCounts( -# survey_tracer="SDSS", -# cluster_abundance=ClusterAbundance( -# halo_mass_definition=pyccl.halos.MassDef(0.5, "matter"), -# halo_mass_function_name="200m", -# halo_mass_function_args={}, -# ), -# cluster_mass=ClusterMassRich(pivot_mass=10.0, pivot_redshift=1.25), -# cluster_redshift=ClusterRedshiftSpec(), -# ) -# return stat - - -# @pytest.fixture(name="missing_survey_tracer") -# def fixture_missing_survey_tracer() -> sacc.Sacc: -# """Return a sacc.Sacc object that lacks a survey_tracer.""" -# return sacc.Sacc() - - -# @pytest.fixture(name="good_sacc_data") -# def fixture_sacc_data(): -# """Return a sacc.Sacc object sufficient to correctly set a -# :python:`ClusterNumberCounts` object. -# """ -# data = sacc.Sacc() -# return data - - -# def test_missing_survey_tracer( -# minimal_stat: ClusterNumberCounts, missing_survey_tracer: sacc.Sacc -# ): -# with pytest.raises( -# ValueError, match="The SACC file does not contain the SurveyTracer SDSS." -# ): -# minimal_stat.read(missing_survey_tracer) - - -# def test_read_works(): -# """After read() is called, we should be able to get the statistic's - -# :python:`DataVector` and also should be able to call -# :python:`compute_theory_vector`. -# """ diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 4d4f3dc00..48df2561d 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -71,15 +71,19 @@ def test_create_musigma_kernel(): def test_cluster_observed_z(): for z in np.geomspace(1.0e-18, 2.0, 20): - f_z = MassRichnessGaussian.observed_value((0.0, 0.0, 1.0), 0.0, z, 0, 0) + z = np.atleast_1d(z) + mass = np.atleast_1d(0) + f_z = MassRichnessGaussian.observed_value((0.0, 0.0, 1.0), mass, z, 0, 0) assert f_z == pytest.approx(np.log1p(z), 1.0e-7, 0.0) def test_cluster_observed_mass(): - for logM in np.linspace(10.0, 16.0, 20): - f_logM = MassRichnessGaussian.observed_value((0.0, 1.0, 0.0), logM, 0.0, 0, 0) + for mass in np.linspace(10.0, 16.0, 20): + z = np.atleast_1d(0) + mass = np.atleast_1d(mass) + f_logM = MassRichnessGaussian.observed_value((0.0, 1.0, 0.0), mass, z, 0, 0) - assert f_logM == pytest.approx(logM * np.log(10.0), 1.0e-7, 0.0) + assert f_logM == pytest.approx(mass * np.log(10.0), 1.0e-7, 0.0) def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned): @@ -126,6 +130,8 @@ def test_cluster_murata_binned_distribution(murata_binned_relation: MurataBinned def test_cluster_murata_binned_mean(murata_binned_relation: MurataBinned): for mass in np.linspace(7.0, 26.0, 20): for z in np.geomspace(1.0e-18, 2.0, 20): + mass = np.atleast_1d(mass) + z = np.atleast_1d(z) test = murata_binned_relation.get_proxy_mean(mass, z) true = MassRichnessGaussian.observed_value( @@ -142,6 +148,8 @@ def test_cluster_murata_binned_mean(murata_binned_relation: MurataBinned): def test_cluster_murata_binned_variance(murata_binned_relation: MurataBinned): for mass in np.linspace(7.0, 26.0, 20): for z in np.geomspace(1.0e-18, 2.0, 20): + mass = np.atleast_1d(mass) + z = np.atleast_1d(z) test = murata_binned_relation.get_proxy_sigma(mass, z) true = MassRichnessGaussian.observed_value( From 0ba105c4b777bfd19e2e5ed032f9ade02ac40180 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 10:53:21 -0700 Subject: [PATCH 39/80] Testing the binned cluster count statistic. --- .../statistic/binned_cluster_number_counts.py | 2 +- .../statistic/test_cluster_number_counts.py | 123 ++++++++++++++++++ tests/test_cluster_data.py | 16 +-- 3 files changed, 132 insertions(+), 9 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 53b7359df..daa6a520f 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -60,7 +60,7 @@ def read(self, sacc_data: sacc.Sacc) -> None: # specify a data type? self.bin_limits = sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) self.data_vector = DataVector.from_list(data_vector) - print(len(data_vector)) + self.sacc_indices = np.array(sacc_indices) super().read(sacc_data) diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index e69de29bb..0f7ab474d 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -0,0 +1,123 @@ +from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( + BinnedClusterNumberCounts, +) +from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator +from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator +from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic +from firecrown.modeling_tools import ModelingTools +from firecrown.models.cluster.abundance import ClusterAbundance +import sacc +import pytest +import pyccl + + +@pytest.fixture(name="sacc_data") +def fixture_complicated_sacc_data(): + cc = sacc.standard_types.cluster_counts + mlm = sacc.standard_types.cluster_mean_log_mass + + s = sacc.Sacc() + s.add_tracer("survey", "my_survey", 4000) + s.add_tracer("survey", "not_my_survey", 5000) + s.add_tracer("bin_z", "my_tracer1", 0, 2) + s.add_tracer("bin_z", "my_tracer2", 2, 4) + s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) + s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) + + s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) + s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) + s.add_data_point(cc, ("not_my_survey", "my_tracer1", "my_other_tracer2"), 1) + + s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer2"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) + + return s + + +def test_create_binned_number_counts(): + integrator = ScipyIntegrator() + bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) + assert bnc is not None + assert bnc.use_cluster_counts is False + assert bnc.use_mean_log_mass is False + assert bnc.survey_name == "Test" + assert bnc.systematics == [] + assert bnc.theory_vector is None + assert len(bnc.data_vector) == 0 + + bnc = BinnedClusterNumberCounts(True, True, "Test", integrator) + assert bnc.use_cluster_counts is True + assert bnc.use_mean_log_mass is True + + systematics = [SourceSystematic("mock_systematic")] + bnc = BinnedClusterNumberCounts(False, False, "Test", integrator, systematics) + assert bnc.systematics == systematics + + +def test_get_data_vector(): + integrator = ScipyIntegrator() + bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) + dv = bnc.get_data_vector() + assert dv is not None + assert len(dv) == 0 + + +def test_read(sacc_data: sacc.Sacc): + integrator = NumCosmoIntegrator() + bnc = BinnedClusterNumberCounts(False, False, "my_survey", integrator) + + with pytest.raises( + RuntimeError, + match="has read a data vector of length 0; the length must be positive", + ): + bnc.read(sacc_data) + + bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) + bnc.read(sacc_data) + assert bnc.sky_area == 4000 + assert len(bnc.bin_limits) == 4 + assert len(bnc.data_vector) == 2 + assert len(bnc.sacc_indices) == 2 + + bnc = BinnedClusterNumberCounts(False, True, "my_survey", integrator) + bnc.read(sacc_data) + assert bnc.sky_area == 4000 + assert len(bnc.bin_limits) == 4 + assert len(bnc.data_vector) == 4 + assert len(bnc.sacc_indices) == 4 + + bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) + bnc.read(sacc_data) + assert bnc.sky_area == 4000 + assert len(bnc.bin_limits) == 4 + assert len(bnc.data_vector) == 6 + assert len(bnc.sacc_indices) == 6 + + +def test_compute_theory_vector(sacc_data: sacc.Sacc): + integrator = NumCosmoIntegrator() + tools = ModelingTools() + hmf = pyccl.halos.MassFuncBocquet16() + cosmo = pyccl.cosmology.CosmologyVanillaLCDM() + tools.cluster_abundance = ClusterAbundance(13, 17, 0, 2, hmf, 4000) + tools.prepare(cosmo) + + bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) + bnc.read(sacc_data) + tv = bnc.compute_theory_vector(tools) + assert tv is not None + assert len(tv) == 4 + + bnc = BinnedClusterNumberCounts(False, True, "my_survey", integrator) + bnc.read(sacc_data) + tv = bnc.compute_theory_vector(tools) + assert tv is not None + assert len(tv) == 8 + + bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) + bnc.read(sacc_data) + tv = bnc.compute_theory_vector(tools) + assert tv is not None + assert len(tv) == 8 diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index b60099c24..23657aa01 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -4,8 +4,8 @@ import sacc -@pytest.fixture -def complicated_sacc_data(): +@pytest.fixture(name="sacc_data") +def fixture_complicated_sacc_data(): cc = sacc.standard_types.cluster_counts mlm = sacc.standard_types.cluster_mean_log_mass @@ -83,8 +83,8 @@ def test_validate_tracers(): ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_counts) -def test_filtered_tracers(complicated_sacc_data): - ad = AbundanceData(complicated_sacc_data, "my_survey", False, False) +def test_filtered_tracers(sacc_data): + ad = AbundanceData(sacc_data, "my_survey", False, False) cc = sacc.standard_types.cluster_counts filtered_tracers, survey_mask = ad.get_filtered_tracers(cc) my_tracers = [ @@ -95,8 +95,8 @@ def test_filtered_tracers(complicated_sacc_data): assert (survey_mask == [True, True, False]).all() -def test_get_data_and_indices(complicated_sacc_data): - ad = AbundanceData(complicated_sacc_data, "my_survey", False, False) +def test_get_data_and_indices(sacc_data): + ad = AbundanceData(sacc_data, "my_survey", False, False) cc = sacc.standard_types.cluster_counts data, indices = ad.get_data_and_indices(cc) @@ -104,8 +104,8 @@ def test_get_data_and_indices(complicated_sacc_data): assert indices == [0, 1] -def test_get_bin_limits(complicated_sacc_data): - ad = AbundanceData(complicated_sacc_data, "my_survey", False, False) +def test_get_bin_limits(sacc_data): + ad = AbundanceData(sacc_data, "my_survey", False, False) cc = sacc.standard_types.cluster_counts limits = ad.get_bin_limits(cc) assert limits == [[(0, 2), (0, 2)], [(0, 2), (2, 4)]] From f41fdac29bae4d23e38de479377d773d6f00103e Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 10:53:39 -0700 Subject: [PATCH 40/80] Single commit for removing mypy linting the tests, not sure if we want to keep this? --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 559b57bba..01569926f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,3 +22,5 @@ explicit_package_bases = True [mypy-firecrown.connector.cobaya.*] disallow_subclassing_any = False +[mypy-tests.*] +ignore_errors = True \ No newline at end of file From c404c5c7202af7cdc636f092b9098fd89ad2c83b Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 11:08:43 -0700 Subject: [PATCH 41/80] Some needed changes after merging with master. Have not modified the cluster code to use new Updatable ModelingTools yet. --- firecrown/modeling_tools.py | 1 - firecrown/models/cluster/kernel.py | 3 +-- firecrown/models/cluster/mass_proxy.py | 24 +++++++++---------- .../statistic/test_cluster_number_counts.py | 7 +++++- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index 38a5385ed..ce13b9cad 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -91,7 +91,6 @@ def prepare( if self.pt_calculator is not None: self.pt_calculator.update_ingredients(ccl_cosmo) - for pkm in self.pk_modifiers: self.add_pk(name=pkm.name, powerspectrum=pkm.compute_p_of_k_z(tools=self)) diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index c7c6871ca..7cba6c628 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,10 +1,9 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import List, Tuple, Union, Optional, Dict, Any +from typing import List, Tuple, Optional import numpy.typing as npt import numpy as np from firecrown.updatable import Updatable -import pdb class KernelType(Enum): diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index c0f3a103e..3fc9e9967 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -109,12 +109,12 @@ def __init__( self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) # Updatable parameters - self.mu_p0 = parameters.create() - self.mu_p1 = parameters.create() - self.mu_p2 = parameters.create() - self.sigma_p0 = parameters.create() - self.sigma_p1 = parameters.create() - self.sigma_p2 = parameters.create() + self.mu_p0 = parameters.register_new_updatable_parameter() + self.mu_p1 = parameters.register_new_updatable_parameter() + self.mu_p2 = parameters.register_new_updatable_parameter() + self.sigma_p0 = parameters.register_new_updatable_parameter() + self.sigma_p1 = parameters.register_new_updatable_parameter() + self.sigma_p2 = parameters.register_new_updatable_parameter() # Verify this gets called last or first @@ -174,12 +174,12 @@ def __init__( self.log1p_pivot_redshift = np.log1p(self.pivot_redshift) # Updatable parameters - self.mu_p0 = parameters.create() - self.mu_p1 = parameters.create() - self.mu_p2 = parameters.create() - self.sigma_p0 = parameters.create() - self.sigma_p1 = parameters.create() - self.sigma_p2 = parameters.create() + self.mu_p0 = parameters.register_new_updatable_parameter() + self.mu_p1 = parameters.register_new_updatable_parameter() + self.mu_p2 = parameters.register_new_updatable_parameter() + self.sigma_p0 = parameters.register_new_updatable_parameter() + self.sigma_p1 = parameters.register_new_updatable_parameter() + self.sigma_p2 = parameters.register_new_updatable_parameter() # Verify this gets called last or first diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 0f7ab474d..b305eefa6 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -5,6 +5,7 @@ from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic from firecrown.modeling_tools import ModelingTools +from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance import sacc import pytest @@ -99,10 +100,14 @@ def test_read(sacc_data: sacc.Sacc): def test_compute_theory_vector(sacc_data: sacc.Sacc): integrator = NumCosmoIntegrator() tools = ModelingTools() + hmf = pyccl.halos.MassFuncBocquet16() cosmo = pyccl.cosmology.CosmologyVanillaLCDM() + params = ParamsMap() + tools.cluster_abundance = ClusterAbundance(13, 17, 0, 2, hmf, 4000) - tools.prepare(cosmo) + tools.update(params) + tools.prepare(cosmo, params) bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) bnc.read(sacc_data) From 4805b6b1309c6e8d241eb523f771e4aad84ad70d Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 2 Nov 2023 13:36:32 -0700 Subject: [PATCH 42/80] Forgot to run flake8! --- .../cluster_redshift_richness.py | 11 ----------- firecrown/connector/cosmosis/likelihood.py | 2 -- tests/test_cluster_abundance.py | 1 - 3 files changed, 14 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index f56f1e86e..332682415 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -6,7 +6,6 @@ import sacc from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator -from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator from firecrown.likelihood.gauss_family.gaussian import ConstGaussian from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( BinnedClusterNumberCounts, @@ -15,9 +14,6 @@ from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import ( - Completeness, - DESY1PhotometricRedshift, - Purity, SpectroscopicRedshift, ) from firecrown.models.cluster.mass_proxy import MurataBinned @@ -39,15 +35,8 @@ def get_cluster_abundance(sky_area: float) -> ClusterAbundance: cluster_abundance.add_kernel(mass_observable_kernel) redshift_proxy_kernel = SpectroscopicRedshift() - # redshift_proxy_kernel = DESY1PhotometricRedshift() cluster_abundance.add_kernel(redshift_proxy_kernel) - # completeness_kernel = Completeness() - # cluster_abundance.add_kernel(completeness_kernel) - - # purity_kernel = Purity() - # cluster_abundance.add_kernel(purity_kernel) - return cluster_abundance diff --git a/firecrown/connector/cosmosis/likelihood.py b/firecrown/connector/cosmosis/likelihood.py index b6ede4f8f..2f2d6fdd3 100644 --- a/firecrown/connector/cosmosis/likelihood.py +++ b/firecrown/connector/cosmosis/likelihood.py @@ -7,8 +7,6 @@ likelihood abstract base class; it the implementation of a CosmoSIS module, not a specific likelihood. """ -import pdb - import cosmosis.datablock from cosmosis.datablock import option_section from cosmosis.datablock import names as section_names diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 211b16b6c..a8af40bd9 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -6,7 +6,6 @@ from typing import List, Tuple, Optional import numpy.typing as npt from firecrown.parameters import ParamsMap, create -import math @pytest.fixture() From 73ca915bd3ce51a87e8e503bb46b120779f439d8 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 12:16:38 -0800 Subject: [PATCH 43/80] Introducing a mock integrator for testing number counts. --- .../statistic/test_cluster_number_counts.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index b305eefa6..7d99c821f 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -1,17 +1,34 @@ from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( BinnedClusterNumberCounts, ) -from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator -from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator +from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic from firecrown.modeling_tools import ModelingTools from firecrown.parameters import ParamsMap -from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand +from typing import Tuple import sacc import pytest import pyccl +class MockIntegrator(Integrator): + def integrate( + self, + integrand: AbundanceIntegrand, + ) -> float: + """Integrate the integrand over the bounds and include extra_args to integral""" + return 1.0 + + def set_integration_bounds( + self, + cl_abundance: ClusterAbundance, + z_proxy_limits: Tuple[float, float], + mass_proxy_limits: Tuple[float, float], + ) -> None: + """Set the limits of integration and extra arguments for the integral""" + + @pytest.fixture(name="sacc_data") def fixture_complicated_sacc_data(): cc = sacc.standard_types.cluster_counts @@ -38,7 +55,7 @@ def fixture_complicated_sacc_data(): def test_create_binned_number_counts(): - integrator = ScipyIntegrator() + integrator = MockIntegrator() bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) assert bnc is not None assert bnc.use_cluster_counts is False @@ -58,7 +75,7 @@ def test_create_binned_number_counts(): def test_get_data_vector(): - integrator = ScipyIntegrator() + integrator = MockIntegrator() bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) dv = bnc.get_data_vector() assert dv is not None @@ -66,7 +83,7 @@ def test_get_data_vector(): def test_read(sacc_data: sacc.Sacc): - integrator = NumCosmoIntegrator() + integrator = MockIntegrator() bnc = BinnedClusterNumberCounts(False, False, "my_survey", integrator) with pytest.raises( @@ -98,7 +115,7 @@ def test_read(sacc_data: sacc.Sacc): def test_compute_theory_vector(sacc_data: sacc.Sacc): - integrator = NumCosmoIntegrator() + integrator = MockIntegrator() tools = ModelingTools() hmf = pyccl.halos.MassFuncBocquet16() From 8b109728a1e0ddc0bd70c735a037a9d92e096c2e Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 12:30:58 -0800 Subject: [PATCH 44/80] Fixing pylint errors. Extracting some methods. --- .../statistic/binned_cluster_number_counts.py | 10 ++- .../cluster/integrator/numcosmo_integrator.py | 24 ++++---- .../cluster/integrator/scipy_integrator.py | 24 ++++---- firecrown/models/cluster/kernel.py | 61 +++++++++++++------ firecrown/models/cluster/mass_proxy.py | 4 +- tests/test_cluster_abundance.py | 18 +++--- tests/test_cluster_integrator_numcosmo.py | 60 +++++++++--------- tests/test_cluster_integrator_scipy.py | 60 +++++++++--------- tests/test_cluster_kernels.py | 10 +-- tests/test_cluster_mass_richness.py | 2 +- 10 files changed, 149 insertions(+), 124 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index daa6a520f..f8c180a5e 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -1,7 +1,8 @@ +"""write me""" from __future__ import annotations -from typing import List, Optional +from typing import List, Optional, Tuple import sacc - +import numpy as np from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.models.cluster.abundance_data import AbundanceData from firecrown.likelihood.gauss_family.statistic.statistic import ( @@ -11,10 +12,11 @@ ) from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic from firecrown.modeling_tools import ModelingTools -import numpy as np class BinnedClusterNumberCounts(Statistic): + """write me""" + def __init__( self, cluster_counts: bool, @@ -31,6 +33,8 @@ def __init__( self.survey_name = survey_name self.integrator = integrator self.data_vector = DataVector.from_list([]) + self.sky_area = 0.0 + self.bin_limits: List[List[Tuple[float, float]]] = [] def read(self, sacc_data: sacc.Sacc) -> None: # Build the data vector and indices needed for the likelihood diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 020a4d425..af4fa249a 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -22,8 +22,8 @@ def __init__( def _default_integral_args(self) -> Dict[KernelType, int]: lkp: Dict[KernelType, int] = dict() - lkp[KernelType.mass] = 0 - lkp[KernelType.z] = 1 + lkp[KernelType.MASS] = 0 + lkp[KernelType.Z] = 1 return lkp def _integral_wrapper( @@ -34,10 +34,10 @@ def _integral_wrapper( def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: default = np.ones_like(int_args[0]) * -1.0 - mass = self._get_or_default(int_args, KernelType.mass, default) - z = self._get_or_default(int_args, KernelType.z, default) - mass_proxy = self._get_or_default(int_args, KernelType.mass_proxy, default) - z_proxy = self._get_or_default(int_args, KernelType.z_proxy, default) + mass = self._get_or_default(int_args, KernelType.MASS, default) + z = self._get_or_default(int_args, KernelType.Z, default) + mass_proxy = self._get_or_default(int_args, KernelType.MASS_PROXY, default) + z_proxy = self._get_or_default(int_args, KernelType.Z_PROXY, default) return_val = integrand( mass, @@ -69,22 +69,22 @@ def set_integration_bounds( self.z_proxy_limits = z_proxy_limits for kernel in cl_abundance.dirac_delta_kernels: - if kernel.kernel_type == KernelType.z_proxy: + if kernel.kernel_type == KernelType.Z_PROXY: self.integral_bounds[1] = z_proxy_limits - elif kernel.kernel_type == KernelType.mass_proxy: + elif kernel.kernel_type == KernelType.MASS_PROXY: self.integral_bounds[0] = mass_proxy_limits for kernel in cl_abundance.integrable_kernels: idx = len(self.integral_bounds) - if kernel.kernel_type == KernelType.z_proxy: + if kernel.kernel_type == KernelType.Z_PROXY: self.integral_bounds.append(z_proxy_limits) - self.integral_args_lkp[KernelType.z_proxy] = idx + self.integral_args_lkp[KernelType.Z_PROXY] = idx - elif kernel.kernel_type == KernelType.mass_proxy: + elif kernel.kernel_type == KernelType.MASS_PROXY: self.integral_bounds.append(mass_proxy_limits) - self.integral_args_lkp[KernelType.mass_proxy] = idx + self.integral_args_lkp[KernelType.MASS_PROXY] = idx return def integrate( diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index d322a2b8f..fccf1b28b 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -22,8 +22,8 @@ def __init__( def _default_integral_args(self) -> Dict[KernelType, int]: lkp: Dict[KernelType, int] = dict() - lkp[KernelType.mass] = 0 - lkp[KernelType.z] = 1 + lkp[KernelType.MASS] = 0 + lkp[KernelType.Z] = 1 return lkp def _integral_wrapper( @@ -33,10 +33,10 @@ def _integral_wrapper( def scipy_integrand(*int_args: float) -> float: default = -1.0 - mass = self._get_or_default(int_args, KernelType.mass, default) - z = self._get_or_default(int_args, KernelType.z, default) - mass_proxy = self._get_or_default(int_args, KernelType.mass_proxy, default) - z_proxy = self._get_or_default(int_args, KernelType.z_proxy, default) + mass = self._get_or_default(int_args, KernelType.MASS, default) + z = self._get_or_default(int_args, KernelType.Z, default) + mass_proxy = self._get_or_default(int_args, KernelType.MASS_PROXY, default) + z_proxy = self._get_or_default(int_args, KernelType.Z_PROXY, default) return_val = integrand( mass, @@ -68,22 +68,22 @@ def set_integration_bounds( self.z_proxy_limits = z_proxy_limits for kernel in cl_abundance.dirac_delta_kernels: - if kernel.kernel_type == KernelType.z_proxy: + if kernel.kernel_type == KernelType.Z_PROXY: self.integral_bounds[1] = z_proxy_limits - elif kernel.kernel_type == KernelType.mass_proxy: + elif kernel.kernel_type == KernelType.MASS_PROXY: self.integral_bounds[0] = mass_proxy_limits for kernel in cl_abundance.integrable_kernels: idx = len(self.integral_bounds) - if kernel.kernel_type == KernelType.z_proxy: + if kernel.kernel_type == KernelType.Z_PROXY: self.integral_bounds.append(z_proxy_limits) - self.integral_args_lkp[KernelType.z_proxy] = idx + self.integral_args_lkp[KernelType.Z_PROXY] = idx - elif kernel.kernel_type == KernelType.mass_proxy: + elif kernel.kernel_type == KernelType.MASS_PROXY: self.integral_bounds.append(mass_proxy_limits) - self.integral_args_lkp[KernelType.mass_proxy] = idx + self.integral_args_lkp[KernelType.MASS_PROXY] = idx return def integrate( diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 7cba6c628..f27dce0a4 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,3 +1,4 @@ +"""write me""" from abc import ABC, abstractmethod from enum import Enum from typing import List, Tuple, Optional @@ -7,15 +8,19 @@ class KernelType(Enum): - mass = 1 - z = 2 - mass_proxy = 3 - z_proxy = 4 - completeness = 5 - purity = 6 + """write me""" + + MASS = 1 + Z = 2 + MASS_PROXY = 3 + Z_PROXY = 4 + COMPLETENESS = 5 + PURITY = 6 class Kernel(Updatable, ABC): + """write me""" + def __init__( self, kernel_type: KernelType, @@ -43,8 +48,10 @@ def distribution( class Completeness(Kernel): + """write me""" + def __init__(self) -> None: - super().__init__(KernelType.completeness) + super().__init__(KernelType.COMPLETENESS) def distribution( self, @@ -67,8 +74,23 @@ def distribution( class Purity(Kernel): + """write me""" + def __init__(self) -> None: - super().__init__(KernelType.purity) + super().__init__(KernelType.PURITY) + + def _ln_rc(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: + a_rc = 2.2183 + b_rc = -0.6592 + ln_rc = a_rc + b_rc * (1.0 + z) + return ln_rc + + def _nc(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: + b_nc = np.log(10) * 0.3527 + a_nc = np.log(10) * 0.8612 + nc = a_nc + b_nc * (1.0 + z) + assert isinstance(nc, np.ndarray) + return nc def distribution( self, @@ -79,29 +101,24 @@ def distribution( mass_proxy_limits: Tuple[float, float], z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: - a_nc = np.log(10) * 0.8612 - b_nc = np.log(10) * 0.3527 - a_rc = 2.2183 - b_rc = -0.6592 - # pdb.set_trace() if all(mass_proxy == -1.0): mean_mass = (mass_proxy_limits[0] + mass_proxy_limits[1]) / 2 ln_r = np.log(10**mean_mass) else: ln_r = np.log(10**mass_proxy) - ln_rc = a_rc + b_rc * (1.0 + z) - r_over_rc = ln_r / ln_rc - nc = a_nc + b_nc * (1.0 + z) + r_over_rc = ln_r / self._ln_rc(z) - purity = (r_over_rc) ** nc / (r_over_rc**nc + 1.0) + purity = (r_over_rc) ** self._nc(z) / (r_over_rc ** self._nc(z) + 1.0) assert isinstance(purity, np.ndarray) return purity class TrueMass(Kernel): + """write me""" + def __init__(self) -> None: - super().__init__(KernelType.mass_proxy, True) + super().__init__(KernelType.MASS_PROXY, True) def distribution( self, @@ -116,8 +133,10 @@ def distribution( class SpectroscopicRedshift(Kernel): + """write me""" + def __init__(self) -> None: - super().__init__(KernelType.z_proxy, True) + super().__init__(KernelType.Z_PROXY, True) def distribution( self, @@ -132,8 +151,10 @@ def distribution( class DESY1PhotometricRedshift(Kernel): + """write me""" + def __init__(self) -> None: - super().__init__(KernelType.z_proxy) + super().__init__(KernelType.Z_PROXY) self.sigma_0 = 0.05 def distribution( diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index 3fc9e9967..ffb13f901 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -102,7 +102,7 @@ def __init__( pivot_redshift: float, integral_bounds: Optional[List[Tuple[float, float]]] = None, ): - super().__init__(KernelType.mass_proxy, False, True, integral_bounds) + super().__init__(KernelType.MASS_PROXY, False, True, integral_bounds) self.pivot_redshift = pivot_redshift self.pivot_mass = pivot_mass * np.log(10.0) # ln(M) @@ -167,7 +167,7 @@ def __init__( pivot_redshift: float, integral_bounds: Optional[List[Tuple[float, float]]] = None, ): - super().__init__(KernelType.mass_proxy, False, True, integral_bounds) + super().__init__(KernelType.MASS_PROXY, False, True, integral_bounds) self.pivot_redshift = pivot_redshift self.pivot_mass = pivot_mass * np.log(10.0) # ln(M) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index a8af40bd9..f85136458 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -53,7 +53,7 @@ def test_cluster_abundance_init(cl_abundance: ClusterAbundance): def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): - mk = MockKernel(KernelType.mass_proxy) + mk = MockKernel(KernelType.MASS_PROXY) cl_abundance.add_kernel(mk) assert cl_abundance.cosmo is None @@ -88,21 +88,21 @@ def test_cluster_sky_area(cl_abundance: ClusterAbundance): def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.kernels) == 0 - cl_abundance.add_kernel(MockKernel(KernelType.mass)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS)) assert len(cl_abundance.kernels) == 1 assert isinstance(cl_abundance.kernels[0], Kernel) - assert cl_abundance.kernels[0].kernel_type == KernelType.mass + assert cl_abundance.kernels[0].kernel_type == KernelType.MASS assert len(cl_abundance.analytic_kernels) == 0 assert len(cl_abundance.dirac_delta_kernels) == 0 assert len(cl_abundance.integrable_kernels) == 1 - cl_abundance.add_kernel(MockKernel(KernelType.mass, True)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS, True)) assert len(cl_abundance.analytic_kernels) == 0 assert len(cl_abundance.dirac_delta_kernels) == 1 assert len(cl_abundance.integrable_kernels) == 1 - cl_abundance.add_kernel(MockKernel(KernelType.mass, False, True)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS, False, True)) assert len(cl_abundance.analytic_kernels) == 1 assert len(cl_abundance.dirac_delta_kernels) == 1 assert len(cl_abundance.integrable_kernels) == 1 @@ -133,7 +133,7 @@ def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): def test_abundance_get_integrand(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.mass)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS)) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) @@ -152,7 +152,7 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.mass)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS)) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) @@ -171,7 +171,7 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.mass)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS)) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) @@ -190,7 +190,7 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.mass)) + cl_abundance.add_kernel(MockKernel(KernelType.MASS)) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) diff --git a/tests/test_cluster_integrator_numcosmo.py b/tests/test_cluster_integrator_numcosmo.py index 0ddead708..6bec20333 100644 --- a/tests/test_cluster_integrator_numcosmo.py +++ b/tests/test_cluster_integrator_numcosmo.py @@ -61,8 +61,8 @@ def test_numcosmo_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundan assert len(nci.integral_bounds) == 2 assert nci.integral_bounds == [(13, 17), (0, 2)] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } @@ -74,7 +74,7 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - dd_kernel = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + dd_kernel = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -85,12 +85,12 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda assert len(nci.integral_bounds) == 2 assert nci.integral_bounds == [mass_limits, (0, 2)] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } cl_abundance.kernels.clear() - dd_kernel = MockKernel(KernelType.z_proxy, is_dirac_delta=True) + dd_kernel = MockKernel(KernelType.Z_PROXY, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -101,10 +101,10 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda assert len(nci.integral_bounds) == 2 assert nci.integral_bounds == [(13, 17), z_limits] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } - dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + dd_kernel2 = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -115,8 +115,8 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda assert len(nci.integral_bounds) == 2 assert nci.integral_bounds == [mass_limits, z_limits] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } @@ -130,7 +130,7 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - ig_kernel = MockKernel(KernelType.mass_proxy) + ig_kernel = MockKernel(KernelType.MASS_PROXY) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -141,13 +141,13 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( assert len(nci.integral_bounds) == 3 assert nci.integral_bounds == [(13, 17), (0, 2), mass_limits] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, - KernelType.mass_proxy: 2, + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.MASS_PROXY: 2, } cl_abundance.kernels.clear() - ig_kernel = MockKernel(KernelType.z_proxy) + ig_kernel = MockKernel(KernelType.Z_PROXY) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -158,12 +158,12 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( assert len(nci.integral_bounds) == 3 assert nci.integral_bounds == [(13, 17), (0, 2), z_limits] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, - KernelType.z_proxy: 2, + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.Z_PROXY: 2, } - ig_kernel2 = MockKernel(KernelType.mass_proxy) + ig_kernel2 = MockKernel(KernelType.MASS_PROXY) cl_abundance.add_kernel(ig_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -174,10 +174,10 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( assert len(nci.integral_bounds) == 4 assert nci.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, - KernelType.z_proxy: 2, - KernelType.mass_proxy: 3, + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.Z_PROXY: 2, + KernelType.MASS_PROXY: 3, } @@ -191,7 +191,7 @@ def test_numcosmo_set_integration_bounds_analytic_slns( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - a_kernel = MockKernel(KernelType.mass_proxy, has_analytic_sln=True) + a_kernel = MockKernel(KernelType.MASS_PROXY, has_analytic_sln=True) cl_abundance.add_kernel(a_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -202,11 +202,11 @@ def test_numcosmo_set_integration_bounds_analytic_slns( assert len(nci.integral_bounds) == 2 assert nci.integral_bounds == [(13, 17), (0, 2)] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } - a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) + a_kernel2 = MockKernel(KernelType.Z_PROXY, has_analytic_sln=True) cl_abundance.add_kernel(a_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -217,8 +217,8 @@ def test_numcosmo_set_integration_bounds_analytic_slns( assert len(nci.integral_bounds) == 2 assert nci.integral_bounds == [(13, 17), (0, 2)] assert nci.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } diff --git a/tests/test_cluster_integrator_scipy.py b/tests/test_cluster_integrator_scipy.py index 7a6b67c4e..4fcf35540 100644 --- a/tests/test_cluster_integrator_scipy.py +++ b/tests/test_cluster_integrator_scipy.py @@ -61,8 +61,8 @@ def test_scipy_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance) assert len(spi.integral_bounds) == 2 assert spi.integral_bounds == [(13, 17), (0, 2)] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } @@ -74,7 +74,7 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - dd_kernel = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + dd_kernel = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -85,12 +85,12 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance assert len(spi.integral_bounds) == 2 assert spi.integral_bounds == [mass_limits, (0, 2)] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } cl_abundance.kernels.clear() - dd_kernel = MockKernel(KernelType.z_proxy, is_dirac_delta=True) + dd_kernel = MockKernel(KernelType.Z_PROXY, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -101,10 +101,10 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance assert len(spi.integral_bounds) == 2 assert spi.integral_bounds == [(13, 17), z_limits] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } - dd_kernel2 = MockKernel(KernelType.mass_proxy, is_dirac_delta=True) + dd_kernel2 = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) cl_abundance.add_kernel(dd_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -115,8 +115,8 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance assert len(spi.integral_bounds) == 2 assert spi.integral_bounds == [mass_limits, z_limits] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } @@ -130,7 +130,7 @@ def test_scipy_set_integration_bounds_integrable_kernels( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - ig_kernel = MockKernel(KernelType.mass_proxy) + ig_kernel = MockKernel(KernelType.MASS_PROXY) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -141,13 +141,13 @@ def test_scipy_set_integration_bounds_integrable_kernels( assert len(spi.integral_bounds) == 3 assert spi.integral_bounds == [(13, 17), (0, 2), mass_limits] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, - KernelType.mass_proxy: 2, + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.MASS_PROXY: 2, } cl_abundance.kernels.clear() - ig_kernel = MockKernel(KernelType.z_proxy) + ig_kernel = MockKernel(KernelType.Z_PROXY) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -158,12 +158,12 @@ def test_scipy_set_integration_bounds_integrable_kernels( assert len(spi.integral_bounds) == 3 assert spi.integral_bounds == [(13, 17), (0, 2), z_limits] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, - KernelType.z_proxy: 2, + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.Z_PROXY: 2, } - ig_kernel2 = MockKernel(KernelType.mass_proxy) + ig_kernel2 = MockKernel(KernelType.MASS_PROXY) cl_abundance.add_kernel(ig_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -174,10 +174,10 @@ def test_scipy_set_integration_bounds_integrable_kernels( assert len(spi.integral_bounds) == 4 assert spi.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, - KernelType.z_proxy: 2, - KernelType.mass_proxy: 3, + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.Z_PROXY: 2, + KernelType.MASS_PROXY: 3, } @@ -191,7 +191,7 @@ def test_scipy_set_integration_bounds_analytic_slns( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - a_kernel = MockKernel(KernelType.mass_proxy, has_analytic_sln=True) + a_kernel = MockKernel(KernelType.MASS_PROXY, has_analytic_sln=True) cl_abundance.add_kernel(a_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -202,11 +202,11 @@ def test_scipy_set_integration_bounds_analytic_slns( assert len(spi.integral_bounds) == 2 assert spi.integral_bounds == [(13, 17), (0, 2)] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } - a_kernel2 = MockKernel(KernelType.z_proxy, has_analytic_sln=True) + a_kernel2 = MockKernel(KernelType.Z_PROXY, has_analytic_sln=True) cl_abundance.add_kernel(a_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -217,8 +217,8 @@ def test_scipy_set_integration_bounds_analytic_slns( assert len(spi.integral_bounds) == 2 assert spi.integral_bounds == [(13, 17), (0, 2)] assert spi.integral_args_lkp == { - KernelType.mass: 0, - KernelType.z: 1, + KernelType.MASS: 0, + KernelType.Z: 1, } diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index d5656ee88..e969691b5 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -14,7 +14,7 @@ def test_create_desy1_photometric_redshift_kernel(): drk = DESY1PhotometricRedshift() assert isinstance(drk, Kernel) - assert drk.kernel_type == KernelType.z_proxy + assert drk.kernel_type == KernelType.Z_PROXY assert drk.is_dirac_delta is False assert drk.integral_bounds is None assert drk.has_analytic_sln is False @@ -23,7 +23,7 @@ def test_create_desy1_photometric_redshift_kernel(): def test_create_spectroscopic_redshift_kernel(): srk = SpectroscopicRedshift() assert isinstance(srk, Kernel) - assert srk.kernel_type == KernelType.z_proxy + assert srk.kernel_type == KernelType.Z_PROXY assert srk.is_dirac_delta is True assert srk.integral_bounds is None assert srk.has_analytic_sln is False @@ -32,7 +32,7 @@ def test_create_spectroscopic_redshift_kernel(): def test_create_mass_kernel(): mk = TrueMass() assert isinstance(mk, Kernel) - assert mk.kernel_type == KernelType.mass_proxy + assert mk.kernel_type == KernelType.MASS_PROXY assert mk.is_dirac_delta is True assert mk.integral_bounds is None assert mk.has_analytic_sln is False @@ -41,7 +41,7 @@ def test_create_mass_kernel(): def test_create_completeness_kernel(): ck = Completeness() assert isinstance(ck, Kernel) - assert ck.kernel_type == KernelType.completeness + assert ck.kernel_type == KernelType.COMPLETENESS assert ck.is_dirac_delta is False assert ck.integral_bounds is None assert ck.has_analytic_sln is False @@ -50,7 +50,7 @@ def test_create_completeness_kernel(): def test_create_purity_kernel(): pk = Purity() assert isinstance(pk, Kernel) - assert pk.kernel_type == KernelType.purity + assert pk.kernel_type == KernelType.PURITY assert pk.is_dirac_delta is False assert pk.integral_bounds is None assert pk.has_analytic_sln is False diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 48df2561d..ea7612450 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -53,7 +53,7 @@ def fixture_murata_unbinned() -> MurataUnbinned: def test_create_musigma_kernel(): mb = MurataBinned(1, 1) assert isinstance(mb, Kernel) - assert mb.kernel_type == KernelType.mass_proxy + assert mb.kernel_type == KernelType.MASS_PROXY assert mb.is_dirac_delta is False assert mb.integral_bounds is None assert mb.has_analytic_sln is True From 8ceae74792711afb1495bf20047a6b1ffd0107fa Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 12:42:12 -0800 Subject: [PATCH 45/80] Pylint changes to integrators. --- firecrown/models/cluster/integrator/integrator.py | 5 ++++- .../cluster/integrator/numcosmo_integrator.py | 12 ++++++++---- .../models/cluster/integrator/scipy_integrator.py | 13 ++++++++----- firecrown/models/cluster/mass_proxy.py | 5 ++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 833831fec..4f197f14a 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -1,9 +1,12 @@ +"""write me""" from abc import ABC, abstractmethod -from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand from typing import Tuple +from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand class Integrator(ABC): + """write me""" + @abstractmethod def integrate( self, diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index af4fa249a..0502ddc80 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -1,13 +1,16 @@ -from numcosmo_py import Ncm -from typing import Tuple, Callable, Dict, Sequence +"""write me""" +from typing import Tuple, Callable, Dict, Sequence, List import numpy as np import numpy.typing as npt +from numcosmo_py import Ncm from firecrown.models.cluster.kernel import KernelType from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand from firecrown.models.cluster.integrator.integrator import Integrator class NumCosmoIntegrator(Integrator): + """write me""" + def __init__( self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 ) -> None: @@ -16,12 +19,13 @@ def __init__( self._absolute_tolerance = absolute_tolerance self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() + self.integral_bounds: List[Tuple[float, float]] = [] self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) def _default_integral_args(self) -> Dict[KernelType, int]: - lkp: Dict[KernelType, int] = dict() + lkp: Dict[KernelType, int] = {} lkp[KernelType.MASS] = 0 lkp[KernelType.Z] = 1 return lkp @@ -85,7 +89,6 @@ def set_integration_bounds( elif kernel.kernel_type == KernelType.MASS_PROXY: self.integral_bounds.append(mass_proxy_limits) self.integral_args_lkp[KernelType.MASS_PROXY] = idx - return def integrate( self, @@ -130,6 +133,7 @@ def __init__( self.dim = dim self.fun = fun + # pylint: disable-next=arguments-differ def do_get_dimensions(self) -> Tuple[int, int]: """Get number of dimensions.""" return self.dim, 1 diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index fccf1b28b..72eaf75c3 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -1,13 +1,16 @@ -from scipy.integrate import nquad -from typing import Callable, Dict, Tuple +"""write me""" +from typing import Callable, Dict, Tuple, List +import numpy as np import numpy.typing as npt +from scipy.integrate import nquad from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.models.cluster.kernel import KernelType from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand -import numpy as np class ScipyIntegrator(Integrator): + """write me""" + def __init__( self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 ) -> None: @@ -15,13 +18,14 @@ def __init__( self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance + self.integral_bounds: List[Tuple[float, float]] = [] self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) def _default_integral_args(self) -> Dict[KernelType, int]: - lkp: Dict[KernelType, int] = dict() + lkp: Dict[KernelType, int] = {} lkp[KernelType.MASS] = 0 lkp[KernelType.Z] = 1 return lkp @@ -84,7 +88,6 @@ def set_integration_bounds( elif kernel.kernel_type == KernelType.MASS_PROXY: self.integral_bounds.append(mass_proxy_limits) self.integral_args_lkp[KernelType.MASS_PROXY] = idx - return def integrate( self, diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index ffb13f901..4eb9997da 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -1,15 +1,18 @@ +"""write me""" from typing import List, Tuple, Optional import numpy as np import numpy.typing as npt from firecrown import parameters -from scipy import special +from scipy import special # pylint: disable=no-member from firecrown.models.cluster.kernel import Kernel, KernelType from abc import abstractmethod class MassRichnessGaussian(Kernel): + """write me""" + @staticmethod def observed_value( p: Tuple[float, float, float], From 2255649b334ef7f0d9a61a9634bd95ba24b98e70 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 13:01:30 -0800 Subject: [PATCH 46/80] Fixing remaining pylint issues, except docstrings. --- .../statistic/binned_cluster_number_counts.py | 4 +++ firecrown/models/cluster/abundance.py | 24 +++++++++++--- firecrown/models/cluster/abundance_data.py | 13 ++++++-- firecrown/models/cluster/mass_proxy.py | 24 ++++++++------ .../statistic/test_cluster_number_counts.py | 19 +++++++---- tests/test_cluster_abundance.py | 28 +++++++++------- tests/test_cluster_data.py | 18 +++++++++-- tests/test_cluster_integrator_numcosmo.py | 32 ++++++++----------- tests/test_cluster_integrator_scipy.py | 32 ++++++++----------- tests/test_cluster_kernels.py | 1 + tests/test_cluster_mass_richness.py | 1 + 11 files changed, 120 insertions(+), 76 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index f8c180a5e..c05e5d333 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -48,11 +48,13 @@ def read(self, sacc_data: sacc.Sacc) -> None: ) if self.use_cluster_counts: + # pylint: disable=no-member data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) data_vector += data sacc_indices += indices if self.use_mean_log_mass: + # pylint: disable=no-member data, indices = sacc_adapter.get_data_and_indices( sacc_types.cluster_mean_log_mass ) @@ -62,6 +64,8 @@ def read(self, sacc_data: sacc.Sacc) -> None: self.sky_area = sacc_adapter.survey_tracer.sky_area # Note - this is the same for both cl mass and cl counts... Why do we need to # specify a data type? + + # pylint: disable=no-member self.bin_limits = sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) self.data_vector = DataVector.from_list(data_vector) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 217a24c81..8361e96b3 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,11 +1,12 @@ +"""write me""" from typing import List, Callable, Optional, Dict, Tuple -from pyccl.cosmology import Cosmology -import pyccl.background as bkg +import numpy as np +import numpy.typing as npt import pyccl +import pyccl.background as bkg +from pyccl.cosmology import Cosmology from firecrown.models.cluster.kernel import Kernel -import numpy as np from firecrown.parameters import ParamsMap -import numpy.typing as npt AbundanceIntegrand = Callable[ @@ -21,9 +22,12 @@ ] -class ClusterAbundance(object): +class ClusterAbundance: + """write me""" + @property def sky_area(self) -> float: + """write me""" return self.sky_area_rad * (180.0 / np.pi) ** 2 @sky_area.setter @@ -32,18 +36,22 @@ def sky_area(self, sky_area: float) -> None: @property def cosmo(self) -> Cosmology: + """write me""" return self._cosmo @property def analytic_kernels(self) -> List[Kernel]: + """write me""" return [x for x in self.kernels if x.has_analytic_sln] @property def dirac_delta_kernels(self) -> List[Kernel]: + """write me""" return [x for x in self.kernels if x.is_dirac_delta] @property def integrable_kernels(self) -> List[Kernel]: + """write me""" return [ x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln ] @@ -68,11 +76,13 @@ def __init__( self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel) -> None: + """write me""" self.kernels.append(kernel) def update_ingredients( self, cosmo: Cosmology, params: Optional[ParamsMap] = None ) -> None: + """write me""" self._cosmo = cosmo self._hmf_cache = {} if params is None: @@ -82,6 +92,7 @@ def update_ingredients( kernel.update(params) def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: + """write me""" scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) @@ -98,6 +109,7 @@ def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64] def mass_function( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] ) -> npt.NDArray[np.float64]: + """write me""" scale_factor = 1.0 / (1.0 + z) return_vals = [] @@ -113,6 +125,8 @@ def mass_function( def get_integrand( self, avg_mass: bool = False, avg_redshift: bool = False ) -> AbundanceIntegrand: + """write me""" + def integrand( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index c8d5ccaef..f871d778b 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -1,11 +1,14 @@ -import sacc -import numpy as np -from sacc.tracers import SurveyTracer +"""write me""" from typing import Tuple, List +import numpy as np import numpy.typing as npt +import sacc +from sacc.tracers import SurveyTracer class AbundanceData: + """write me""" + # Hard coded in SACC, how do we want to handle this? _survey_index = 0 _redshift_index = 1 @@ -32,6 +35,7 @@ def __init__( raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") def get_filtered_tracers(self, data_type: str) -> Tuple[npt.NDArray, npt.NDArray]: + """write me""" all_tracers = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) @@ -41,6 +45,7 @@ def get_filtered_tracers(self, data_type: str) -> Tuple[npt.NDArray, npt.NDArray return filtered_tracers, survey_mask def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: + """write me""" _, survey_mask = self.get_filtered_tracers(data_type) data_vector_list = list( self.sacc_data.get_mean(data_type=data_type)[survey_mask] @@ -53,6 +58,7 @@ def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: def validate_tracers( self, tracers_combinations: npt.NDArray, data_type: str ) -> None: + """write me""" if len(tracers_combinations) == 0: raise ValueError( f"The SACC file does not contain any tracers for the " @@ -67,6 +73,7 @@ def validate_tracers( ) def get_bin_limits(self, data_type: str) -> List[List[Tuple[float, float]]]: + """write me""" filtered_tracers, _ = self.get_filtered_tracers(data_type) tracers = [] diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index 4eb9997da..6ca79c7c7 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -1,13 +1,11 @@ """write me""" from typing import List, Tuple, Optional - +from abc import abstractmethod import numpy as np import numpy.typing as npt - +from scipy import special from firecrown import parameters -from scipy import special # pylint: disable=no-member from firecrown.models.cluster.kernel import Kernel, KernelType -from abc import abstractmethod class MassRichnessGaussian(Kernel): @@ -51,10 +49,10 @@ def _distribution_binned( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: proxy_mean = self.get_proxy_mean(mass, z) proxy_sigma = self.get_proxy_sigma(mass, z) @@ -70,9 +68,11 @@ def _distribution_binned( mask1 = (x_max > 3.0) | (x_min < -3.0) mask2 = ~mask1 + # pylint: disable=no-member return_vals[mask1] = ( -(special.erfc(x_min[mask1]) - special.erfc(x_max[mask1])) / 2.0 ) + # pylint: disable=no-member return_vals[mask2] = ( special.erf(x_min[mask2]) - special.erf(x_max[mask2]) ) / 2.0 @@ -84,9 +84,9 @@ def _distribution_unbinned( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: proxy_mean = self.get_proxy_mean(mass, z) proxy_sigma = self.get_proxy_sigma(mass, z) @@ -99,6 +99,8 @@ def _distribution_unbinned( class MurataBinned(MassRichnessGaussian): + """write me""" + def __init__( self, pivot_mass: float, @@ -164,6 +166,8 @@ def distribution( class MurataUnbinned(MassRichnessGaussian): + """write me""" + def __init__( self, pivot_mass: float, diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 7d99c821f..22d5fd8d1 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -1,18 +1,21 @@ -from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( - BinnedClusterNumberCounts, -) +"""write me""" +from typing import Tuple +import sacc +import pytest +import pyccl from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic from firecrown.modeling_tools import ModelingTools from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand -from typing import Tuple -import sacc -import pytest -import pyccl +from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( + BinnedClusterNumberCounts, +) class MockIntegrator(Integrator): + """write me""" + def integrate( self, integrand: AbundanceIntegrand, @@ -31,7 +34,9 @@ def set_integration_bounds( @pytest.fixture(name="sacc_data") def fixture_complicated_sacc_data(): + # pylint: disable=no-member cc = sacc.standard_types.cluster_counts + # pylint: disable=no-member mlm = sacc.standard_types.cluster_mean_log_mass s = sacc.Sacc() diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index f85136458..b907c258f 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,21 +1,25 @@ +"""write me""" +from typing import List, Tuple, Optional import pytest -from firecrown.models.cluster.abundance import ClusterAbundance -from firecrown.models.cluster.kernel import Kernel, KernelType import pyccl import numpy as np -from typing import List, Tuple, Optional import numpy.typing as npt from firecrown.parameters import ParamsMap, create +from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.kernel import Kernel, KernelType -@pytest.fixture() -def cl_abundance(): +@pytest.fixture(name="cl_abundance") +def fixture_cl_abundance(): + """write me""" hmf = pyccl.halos.MassFuncBocquet16() ca = ClusterAbundance(13, 17, 0, 2, hmf, 360.0**2) return ca class MockKernel(Kernel): + """write me""" + def __init__( self, kernel_type: KernelType, @@ -28,12 +32,12 @@ def __init__( def distribution( self, - mass: npt.NDArray[np.float64], - z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass: npt.NDArray[np.float64], + _z: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: """The functional form of the distribution or spread of this kernel""" return np.atleast_1d(1.0) @@ -58,6 +62,7 @@ def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): assert cl_abundance.cosmo is None assert mk.param is None + # pylint: disable=protected-access assert cl_abundance._hmf_cache == {} pmap = ParamsMap({"param": 42}) @@ -67,6 +72,7 @@ def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): assert cl_abundance.cosmo is not None assert cl_abundance.cosmo == cosmo assert mk.param == 42 + # pylint: disable=protected-access assert cl_abundance._hmf_cache == {} cl_abundance.update_ingredients(None) diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index 23657aa01..ffed8705a 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -1,12 +1,15 @@ +"""write me""" import pytest import numpy as np -from firecrown.models.cluster.abundance_data import AbundanceData import sacc +from firecrown.models.cluster.abundance_data import AbundanceData @pytest.fixture(name="sacc_data") def fixture_complicated_sacc_data(): + # pylint: disable=no-member cc = sacc.standard_types.cluster_counts + # pylint: disable=no-member mlm = sacc.standard_types.cluster_mean_log_mass s = sacc.Sacc() @@ -52,8 +55,11 @@ def test_create_abundance_data(): assert ad.mean_log_mass is True assert ad.survey_nm == "mock_survey" assert ad.survey_tracer.sky_area == 4000 + # pylint: disable=protected-access assert ad._mass_index == 2 + # pylint: disable=protected-access assert ad._redshift_index == 1 + # pylint: disable=protected-access assert ad._survey_index == 0 @@ -61,30 +67,36 @@ def test_validate_tracers(): s = sacc.Sacc() s.add_tracer("survey", "mock_survey", 4000) s.add_tracer("bin_z", "my_tracer", 0, 2) + # pylint: disable=no-member s.add_data_point( sacc.standard_types.cluster_counts, ("mock_survey", "my_tracer"), 1 ) ad = AbundanceData(s, "mock_survey", False, False) - + # pylint: disable=no-member tracer_combs = np.array( s.get_tracer_combinations(sacc.standard_types.cluster_mean_log_mass) ) + # pylint: disable=no-member with pytest.raises( ValueError, match="The SACC file does not contain any tracers for the" + f" {sacc.standard_types.cluster_mean_log_mass} data type", ): + # pylint: disable=no-member ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_mean_log_mass) + # pylint: disable=no-member tracer_combs = np.array( s.get_tracer_combinations(sacc.standard_types.cluster_counts) ) with pytest.raises(ValueError, match="The SACC file must contain 3 tracers"): + # pylint: disable=no-member ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_counts) def test_filtered_tracers(sacc_data): ad = AbundanceData(sacc_data, "my_survey", False, False) + # pylint: disable=no-member cc = sacc.standard_types.cluster_counts filtered_tracers, survey_mask = ad.get_filtered_tracers(cc) my_tracers = [ @@ -97,6 +109,7 @@ def test_filtered_tracers(sacc_data): def test_get_data_and_indices(sacc_data): ad = AbundanceData(sacc_data, "my_survey", False, False) + # pylint: disable=no-member cc = sacc.standard_types.cluster_counts data, indices = ad.get_data_and_indices(cc) @@ -106,6 +119,7 @@ def test_get_data_and_indices(sacc_data): def test_get_bin_limits(sacc_data): ad = AbundanceData(sacc_data, "my_survey", False, False) + # pylint: disable=no-member cc = sacc.standard_types.cluster_counts limits = ad.get_bin_limits(cc) assert limits == [[(0, 2), (0, 2)], [(0, 2), (2, 4)]] diff --git a/tests/test_cluster_integrator_numcosmo.py b/tests/test_cluster_integrator_numcosmo.py index 6bec20333..5868d1f67 100644 --- a/tests/test_cluster_integrator_numcosmo.py +++ b/tests/test_cluster_integrator_numcosmo.py @@ -1,7 +1,8 @@ +"""write me""" +from typing import Tuple import numpy as np import numpy.typing as npt import pytest -from typing import List, Tuple, Optional from firecrown.models.cluster.integrator.numcosmo_integrator import ( NumCosmoIntegrator, ) @@ -23,23 +24,16 @@ def fixture_cl_abundance(): class MockKernel(Kernel): - def __init__( - self, - kernel_type: KernelType, - is_dirac_delta: bool = False, - has_analytic_sln: bool = False, - integral_bounds: Optional[List[Tuple[float, float]]] = None, - ): - super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) + """write me""" def distribution( self, - mass: npt.NDArray[np.float64], - z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass: npt.NDArray[np.float64], + _z: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: """The functional form of the distribution or spread of this kernel""" return np.atleast_1d(1.0) @@ -232,10 +226,10 @@ def test_numcosmo_integrator_integrate(cl_abundance: ClusterAbundance): def integrand( a: npt.NDArray[np.float64], b: npt.NDArray[np.float64], - c: npt.NDArray[np.float64], - d: npt.NDArray[np.float64], - e: Tuple[float, float], - f: Tuple[float, float], + _c: npt.NDArray[np.float64], + _d: npt.NDArray[np.float64], + _e: Tuple[float, float], + _f: Tuple[float, float], ): # xy result = a * b diff --git a/tests/test_cluster_integrator_scipy.py b/tests/test_cluster_integrator_scipy.py index 4fcf35540..b0431c0b0 100644 --- a/tests/test_cluster_integrator_scipy.py +++ b/tests/test_cluster_integrator_scipy.py @@ -1,7 +1,8 @@ +"""write me""" +from typing import Tuple import numpy as np import numpy.typing as npt import pytest -from typing import List, Tuple, Optional from firecrown.models.cluster.integrator.scipy_integrator import ( ScipyIntegrator, ) @@ -23,23 +24,16 @@ def fixture_cl_abundance(): class MockKernel(Kernel): - def __init__( - self, - kernel_type: KernelType, - is_dirac_delta: bool = False, - has_analytic_sln: bool = False, - integral_bounds: Optional[List[Tuple[float, float]]] = None, - ): - super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) + """write me""" def distribution( self, - mass: npt.NDArray[np.float64], - z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass: npt.NDArray[np.float64], + _z: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: """The functional form of the distribution or spread of this kernel""" return np.atleast_1d(1.0) @@ -232,10 +226,10 @@ def test_scipy_integrator_integrate(cl_abundance: ClusterAbundance): def integrand( a: npt.NDArray[np.float64], b: npt.NDArray[np.float64], - c: npt.NDArray[np.float64], - d: npt.NDArray[np.float64], - e: Tuple[float, float], - f: Tuple[float, float], + _c: npt.NDArray[np.float64], + _d: npt.NDArray[np.float64], + _e: Tuple[float, float], + _f: Tuple[float, float], ): # xy result = a * b diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index e969691b5..9d757592a 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -1,3 +1,4 @@ +"""write me""" import pytest import numpy as np from firecrown.models.cluster.kernel import ( diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index ea7612450..0e64ceda2 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,3 +1,4 @@ +"""write me""" import pytest import numpy as np from firecrown.models.cluster.mass_proxy import ( From b8696b73dd60dd204905814ed9f3f598c5570e68 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 14:21:19 -0800 Subject: [PATCH 47/80] Filling in doc strings. --- firecrown/models/cluster/abundance.py | 39 +++++++++++++------ .../models/cluster/integrator/integrator.py | 20 ++++++++-- .../cluster/integrator/numcosmo_integrator.py | 15 +++---- .../cluster/integrator/scipy_integrator.py | 8 ++-- 4 files changed, 56 insertions(+), 26 deletions(-) diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 8361e96b3..08c019631 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -1,4 +1,9 @@ -"""write me""" +"""The module responsible for building the cluster abundance calculation. + +The galaxy cluster abundance integral is a combination of both theoretical +and phenomenological predictions. This module contains the classes and +functions that produce those predictions. +""" from typing import List, Callable, Optional, Dict, Tuple import numpy as np import numpy.typing as npt @@ -23,11 +28,21 @@ class ClusterAbundance: - """write me""" + """The class that calculates the predicted number counts of galaxy clusters + + The abundance is a function of a specific cosmology, a mass and redshift range, + an area on the sky, a halo mass function, as well as multiple kernels, where + each kernel represents a different distribution involved in the final cluster + abundance integrand. + """ @property def sky_area(self) -> float: - """write me""" + """The sky area in square degrees + + The cluster number count prediction will be different depending on + the area of the sky we are considering. This value controls that. + """ return self.sky_area_rad * (180.0 / np.pi) ** 2 @sky_area.setter @@ -36,22 +51,22 @@ def sky_area(self, sky_area: float) -> None: @property def cosmo(self) -> Cosmology: - """write me""" + """The cosmology used to predict the cluster number count.""" return self._cosmo @property def analytic_kernels(self) -> List[Kernel]: - """write me""" + """The kernels in in the integrand which have an analytic solution.""" return [x for x in self.kernels if x.has_analytic_sln] @property def dirac_delta_kernels(self) -> List[Kernel]: - """write me""" + """The kernels in in the integrand which are dirac delta functions.""" return [x for x in self.kernels if x.is_dirac_delta] @property def integrable_kernels(self) -> List[Kernel]: - """write me""" + """The kernels in in the integrand which must be integrated.""" return [ x for x in self.kernels if not x.is_dirac_delta and not x.has_analytic_sln ] @@ -76,13 +91,13 @@ def __init__( self._cosmo: Cosmology = None def add_kernel(self, kernel: Kernel) -> None: - """write me""" + """Add a kernel to the cluster abundance integrand""" self.kernels.append(kernel) def update_ingredients( self, cosmo: Cosmology, params: Optional[ParamsMap] = None ) -> None: - """write me""" + """Update the cluster abundance calculation with a new cosmology.""" self._cosmo = cosmo self._hmf_cache = {} if params is None: @@ -92,7 +107,7 @@ def update_ingredients( kernel.update(params) def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: - """write me""" + """The differential comoving volume at redshift z.""" scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) @@ -109,7 +124,7 @@ def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64] def mass_function( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] ) -> npt.NDArray[np.float64]: - """write me""" + """The mass function at z and mass.""" scale_factor = 1.0 / (1.0 + z) return_vals = [] @@ -125,7 +140,7 @@ def mass_function( def get_integrand( self, avg_mass: bool = False, avg_redshift: bool = False ) -> AbundanceIntegrand: - """write me""" + """Returns a callable that evaluates the complete integrand.""" def integrand( mass: npt.NDArray[np.float64], diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 4f197f14a..466790709 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -1,18 +1,26 @@ -"""write me""" +"""The integrator module + +This module holds the classes that define the interface required to +integrate an assembled cluster abundance. +""" + from abc import ABC, abstractmethod from typing import Tuple from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand class Integrator(ABC): - """write me""" + """The integrator base class + + This class acts as an adapter around an integration library, and must provides + a specific set of methods to be used to integrate a cluster abundance integral.""" @abstractmethod def integrate( self, integrand: AbundanceIntegrand, ) -> float: - """Integrate the integrand over the bounds and include extra_args to integral""" + """Call this method to integrate the provided integrand argument.""" @abstractmethod def set_integration_bounds( @@ -21,4 +29,8 @@ def set_integration_bounds( z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: - """Set the limits of integration and extra arguments for the integral""" + """Sets the limits of integration used by the integration library. + + This method uses the mass and redshift proxy arguments, along with + the cluster abundance argument to construct the limits of integration + to be used in evaluating the cluster abundance.""" diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 0502ddc80..32e1050e4 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -1,4 +1,7 @@ -"""write me""" +"""The NumCosmo integrator module + +This module holds the NumCosmo implementation of the integrator classes +""" from typing import Tuple, Callable, Dict, Sequence, List import numpy as np import numpy.typing as npt @@ -9,7 +12,7 @@ class NumCosmoIntegrator(Integrator): - """write me""" + """The NumCosmo implementation of the Integrator base class.""" def __init__( self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 @@ -62,7 +65,6 @@ def set_integration_bounds( z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: - # pdb.set_trace() self.integral_args_lkp = self._default_integral_args() self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), @@ -121,8 +123,7 @@ def _get_or_default( class CountsIntegralND(Ncm.IntegralND): - """Integral subclass used by the ClusterAbundance - to compute the integrals using numcosmo.""" + """Integral subclass used to compute the integrals using NumCosmo.""" def __init__( self, @@ -135,7 +136,7 @@ def __init__( # pylint: disable-next=arguments-differ def do_get_dimensions(self) -> Tuple[int, int]: - """Get number of dimensions.""" + """Returns the dimensionality of the integral.""" return self.dim, 1 # pylint: disable-next=arguments-differ @@ -147,6 +148,6 @@ def do_integrand( _fdim: int, fval_vec: Ncm.Vector, ) -> None: - """Integrand function.""" + """Called by NumCosmo to evaluate the integrand.""" x = np.array(x_vec.dup_array()).reshape(npoints, dim) fval_vec.set_array(self.fun(x)) diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index 72eaf75c3..bb4b27536 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -1,4 +1,7 @@ -"""write me""" +"""The SciPy integrator module + +This module holds the scipy implementation of the integrator classes +""" from typing import Callable, Dict, Tuple, List import numpy as np import numpy.typing as npt @@ -9,7 +12,7 @@ class ScipyIntegrator(Integrator): - """write me""" + """The scipy implementation of the Integrator base class using nquad.""" def __init__( self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 @@ -61,7 +64,6 @@ def set_integration_bounds( z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: - # pdb.set_trace() self.integral_args_lkp = self._default_integral_args() self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), From 39e563f7b0ae500470c582ca1e7d6e08c61dc9f9 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 14:48:23 -0800 Subject: [PATCH 48/80] Finishing docstrings for cluster module. --- .../statistic/binned_cluster_number_counts.py | 12 ++- firecrown/models/cluster/abundance_data.py | 18 ++-- firecrown/models/cluster/kernel.py | 82 ++++++++++++------- firecrown/models/cluster/mass_proxy.py | 16 ++-- 4 files changed, 81 insertions(+), 47 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index c05e5d333..a6eebbd1e 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -1,4 +1,8 @@ -"""write me""" +"""This module holds classes needed to predict the binned cluster number counts + +The binned cluster number counts statistic predicts the number of galaxy +clusters within a single redshift and mass bin. +""" from __future__ import annotations from typing import List, Optional, Tuple import sacc @@ -15,7 +19,11 @@ class BinnedClusterNumberCounts(Statistic): - """write me""" + """The Binned Cluster Number Counts statistic + + This class will make a prediction for the number of clusters in a z, mass bin + and compare that prediction to the data provided in the sacc file. + """ def __init__( self, diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index f871d778b..754a363f3 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -1,4 +1,5 @@ -"""write me""" +"""The module responsible for extracting cluster data from a sacc file. +""" from typing import Tuple, List import numpy as np import numpy.typing as npt @@ -7,7 +8,12 @@ class AbundanceData: - """write me""" + """The class used to wrap a sacc file and return the cluster abundance data. + + The sacc file is a complicated set of tracers and surveys. This class + manipulates that data and returns only the data relevant for the cluster + number count statistic. The data in this class is specific to a single + survey name.""" # Hard coded in SACC, how do we want to handle this? _survey_index = 0 @@ -35,7 +41,7 @@ def __init__( raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") def get_filtered_tracers(self, data_type: str) -> Tuple[npt.NDArray, npt.NDArray]: - """write me""" + """Returns only tracers that match the data type requested.""" all_tracers = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) @@ -45,7 +51,7 @@ def get_filtered_tracers(self, data_type: str) -> Tuple[npt.NDArray, npt.NDArray return filtered_tracers, survey_mask def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: - """write me""" + """Returns the data vector and indices for the requested data type.""" _, survey_mask = self.get_filtered_tracers(data_type) data_vector_list = list( self.sacc_data.get_mean(data_type=data_type)[survey_mask] @@ -58,7 +64,7 @@ def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: def validate_tracers( self, tracers_combinations: npt.NDArray, data_type: str ) -> None: - """write me""" + """Validates that the tracers requested exist and are valid.""" if len(tracers_combinations) == 0: raise ValueError( f"The SACC file does not contain any tracers for the " @@ -73,7 +79,7 @@ def validate_tracers( ) def get_bin_limits(self, data_type: str) -> List[List[Tuple[float, float]]]: - """write me""" + """Returns the limits for all z, mass bins for the requested data type.""" filtered_tracers, _ = self.get_filtered_tracers(data_type) tracers = [] diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index f27dce0a4..31c194273 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -1,4 +1,7 @@ -"""write me""" +"""The cluster kernel module + +This module holds the classes that define the kernels that can be included +in the cluster abundance integrand.""" from abc import ABC, abstractmethod from enum import Enum from typing import List, Tuple, Optional @@ -8,7 +11,7 @@ class KernelType(Enum): - """write me""" + """The kernels that can be included in the cluster abundance integrand""" MASS = 1 Z = 2 @@ -19,7 +22,10 @@ class KernelType(Enum): class Kernel(Updatable, ABC): - """write me""" + """The Kernel base class + + This class defines the common interface that any kernel must implement in order + to be included in the cluster abundance integrand.""" def __init__( self, @@ -48,7 +54,10 @@ def distribution( class Completeness(Kernel): - """write me""" + """The completeness kernel + + This kernel will affect the integrand by accounting for the incompleteness + of a cluster selection.""" def __init__(self) -> None: super().__init__(KernelType.COMPLETENESS) @@ -57,10 +66,10 @@ def distribution( self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: a_nc = 1.1321 b_nc = 0.7751 @@ -74,7 +83,10 @@ def distribution( class Purity(Kernel): - """write me""" + """The purity kernel + + This kernel will affect the integrand by accounting for the purity + of a cluster selection.""" def __init__(self) -> None: super().__init__(KernelType.PURITY) @@ -94,12 +106,12 @@ def _nc(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: def distribution( self, - mass: npt.NDArray[np.float64], + _mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: if all(mass_proxy == -1.0): mean_mass = (mass_proxy_limits[0] + mass_proxy_limits[1]) / 2 @@ -115,43 +127,51 @@ def distribution( class TrueMass(Kernel): - """write me""" + """The true mass kernel. + + Assuming we measure the true mass, this will always be 1.""" def __init__(self) -> None: super().__init__(KernelType.MASS_PROXY, True) def distribution( self, - mass: npt.NDArray[np.float64], - z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass: npt.NDArray[np.float64], + _z: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: return np.atleast_1d(1.0) class SpectroscopicRedshift(Kernel): - """write me""" + """The spec-z kernel. + + Assuming the spectroscopic redshift has no uncertainties, this is akin to + multiplying by 1.""" def __init__(self) -> None: super().__init__(KernelType.Z_PROXY, True) def distribution( self, - mass: npt.NDArray[np.float64], - z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass: npt.NDArray[np.float64], + _z: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], + _z_proxy: npt.NDArray[np.float64], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: return np.atleast_1d(1.0) class DESY1PhotometricRedshift(Kernel): - """write me""" + """The DES Y1 photometric redshift uncertainties kernel. + + This kernel includes a spread in the photo-z estimates following the + observed spread in DES Y1.""" def __init__(self) -> None: super().__init__(KernelType.Z_PROXY) @@ -159,12 +179,12 @@ def __init__(self) -> None: def distribution( self, - mass: npt.NDArray[np.float64], + _mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], - mass_proxy: npt.NDArray[np.float64], + _mass_proxy: npt.NDArray[np.float64], z_proxy: npt.NDArray[np.float64], - mass_proxy_limits: Tuple[float, float], - z_proxy_limits: Tuple[float, float], + _mass_proxy_limits: Tuple[float, float], + _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: sigma_z = self.sigma_0 * (1 + z) prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index 6ca79c7c7..cd35689be 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -1,4 +1,8 @@ -"""write me""" +"""The mass richness kernel module + +This module holds the classes that define the mass richness relations +that can be included in the cluster abundance integrand. These are +implementations of Kernels.""" from typing import List, Tuple, Optional from abc import abstractmethod import numpy as np @@ -9,7 +13,7 @@ class MassRichnessGaussian(Kernel): - """write me""" + """The representation of mass richness relations that are of a gaussian form.""" @staticmethod def observed_value( @@ -99,7 +103,7 @@ def _distribution_unbinned( class MurataBinned(MassRichnessGaussian): - """write me""" + """The mass richness relation defined in Murata 19 for a binned data vector.""" def __init__( self, @@ -128,7 +132,6 @@ def get_proxy_mean( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], ) -> npt.NDArray[np.float64]: - """Return observed quantity corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), mass, @@ -142,7 +145,6 @@ def get_proxy_sigma( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], ) -> npt.NDArray[np.float64]: - """Return observed scatter corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.sigma_p0, self.sigma_p1, self.sigma_p2), mass, @@ -166,7 +168,7 @@ def distribution( class MurataUnbinned(MassRichnessGaussian): - """write me""" + """The mass richness relation defined in Murata 19 for a unbinned data vector.""" def __init__( self, @@ -195,7 +197,6 @@ def get_proxy_mean( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], ) -> npt.NDArray[np.float64]: - """Return observed quantity corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.mu_p0, self.mu_p1, self.mu_p2), mass, @@ -209,7 +210,6 @@ def get_proxy_sigma( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], ) -> npt.NDArray[np.float64]: - """Return observed scatter corrected by redshift and mass.""" return MassRichnessGaussian.observed_value( (self.sigma_p0, self.sigma_p1, self.sigma_p2), mass, From 78e1e61b6e411f2af6af29214be40e8955819464 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 14:51:18 -0800 Subject: [PATCH 49/80] Docstrings for tests. --- .../gauss_family/statistic/test_cluster_number_counts.py | 4 ++-- tests/test_cluster_abundance.py | 6 +++--- tests/test_cluster_data.py | 2 +- tests/test_cluster_integrator_numcosmo.py | 4 ++-- tests/test_cluster_integrator_scipy.py | 4 ++-- tests/test_cluster_kernels.py | 2 +- tests/test_cluster_mass_richness.py | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 22d5fd8d1..83618d234 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the binned cluster number counts module.""" from typing import Tuple import sacc import pytest @@ -14,7 +14,7 @@ class MockIntegrator(Integrator): - """write me""" + """A mock integrator used by the cluster number counts statistic for testing""" def integrate( self, diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index b907c258f..ae89d95dd 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the cluster abundance module.""" from typing import List, Tuple, Optional import pytest import pyccl @@ -11,14 +11,14 @@ @pytest.fixture(name="cl_abundance") def fixture_cl_abundance(): - """write me""" + """Test fixture that represents an assembled cluster abundance class.""" hmf = pyccl.halos.MassFuncBocquet16() ca = ClusterAbundance(13, 17, 0, 2, hmf, 360.0**2) return ca class MockKernel(Kernel): - """write me""" + """A mock implementation of a Kernel used for unit testing.""" def __init__( self, diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index ffed8705a..af345dc0e 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the cluster abundance data module.""" import pytest import numpy as np import sacc diff --git a/tests/test_cluster_integrator_numcosmo.py b/tests/test_cluster_integrator_numcosmo.py index 5868d1f67..d538fdcea 100644 --- a/tests/test_cluster_integrator_numcosmo.py +++ b/tests/test_cluster_integrator_numcosmo.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the numcosmo integrator module.""" from typing import Tuple import numpy as np import numpy.typing as npt @@ -24,7 +24,7 @@ def fixture_cl_abundance(): class MockKernel(Kernel): - """write me""" + """A mock implementation of a Kernel used for unit testing.""" def distribution( self, diff --git a/tests/test_cluster_integrator_scipy.py b/tests/test_cluster_integrator_scipy.py index b0431c0b0..654e7522e 100644 --- a/tests/test_cluster_integrator_scipy.py +++ b/tests/test_cluster_integrator_scipy.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the scipy integrator module.""" from typing import Tuple import numpy as np import numpy.typing as npt @@ -24,7 +24,7 @@ def fixture_cl_abundance(): class MockKernel(Kernel): - """write me""" + """A mock implementation of a Kernel used for unit testing.""" def distribution( self, diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 9d757592a..aec37bcd1 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the cluster kernel module.""" import pytest import numpy as np from firecrown.models.cluster.kernel import ( diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index 0e64ceda2..c8e7fc532 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,4 +1,4 @@ -"""write me""" +"""Tests for the cluster mass richness module.""" import pytest import numpy as np from firecrown.models.cluster.mass_proxy import ( From 9387cb10aa17b126811dac2f70e17ecc5e088790 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 7 Nov 2023 14:59:47 -0800 Subject: [PATCH 50/80] Extract method refactoring based on pylint suggestion. --- firecrown/models/cluster/integrator/integrator.py | 10 ++++++++-- .../models/cluster/integrator/numcosmo_integrator.py | 9 ++------- .../models/cluster/integrator/scipy_integrator.py | 9 ++------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 466790709..539bc8330 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -3,10 +3,10 @@ This module holds the classes that define the interface required to integrate an assembled cluster abundance. """ - from abc import ABC, abstractmethod -from typing import Tuple +from typing import Tuple, Dict from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand +from firecrown.models.cluster.kernel import KernelType class Integrator(ABC): @@ -34,3 +34,9 @@ def set_integration_bounds( This method uses the mass and redshift proxy arguments, along with the cluster abundance argument to construct the limits of integration to be used in evaluating the cluster abundance.""" + + def _default_integral_args(self) -> Dict[KernelType, int]: + lkp: Dict[KernelType, int] = {} + lkp[KernelType.MASS] = 0 + lkp[KernelType.Z] = 1 + return lkp diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 32e1050e4..dffb4adb5 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -27,12 +27,6 @@ def __init__( self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) - def _default_integral_args(self) -> Dict[KernelType, int]: - lkp: Dict[KernelType, int] = {} - lkp[KernelType.MASS] = 0 - lkp[KernelType.Z] = 1 - return lkp - def _integral_wrapper( self, integrand: AbundanceIntegrand, @@ -40,7 +34,7 @@ def _integral_wrapper( # mypy strict issue: npt.NDArray[npt.NDArray[np.float64]] not supported def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: default = np.ones_like(int_args[0]) * -1.0 - + # pylint: disable=R0801 mass = self._get_or_default(int_args, KernelType.MASS, default) z = self._get_or_default(int_args, KernelType.Z, default) mass_proxy = self._get_or_default(int_args, KernelType.MASS_PROXY, default) @@ -65,6 +59,7 @@ def set_integration_bounds( z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: + # pylint: disable=R0801 self.integral_args_lkp = self._default_integral_args() self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index bb4b27536..1fdbbbc9d 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -27,19 +27,13 @@ def __init__( self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) - def _default_integral_args(self) -> Dict[KernelType, int]: - lkp: Dict[KernelType, int] = {} - lkp[KernelType.MASS] = 0 - lkp[KernelType.Z] = 1 - return lkp - def _integral_wrapper( self, integrand: AbundanceIntegrand, ) -> Callable[..., float]: def scipy_integrand(*int_args: float) -> float: default = -1.0 - + # pylint: disable=R0801 mass = self._get_or_default(int_args, KernelType.MASS, default) z = self._get_or_default(int_args, KernelType.Z, default) mass_proxy = self._get_or_default(int_args, KernelType.MASS_PROXY, default) @@ -64,6 +58,7 @@ def set_integration_bounds( z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: + # pylint: disable=R0801 self.integral_args_lkp = self._default_integral_args() self.integral_bounds = [ (cl_abundance.min_mass, cl_abundance.max_mass), From a30f95930a7ed2690f7d315930375a83892f0cfe Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 11:42:01 -0800 Subject: [PATCH 51/80] Moving cluster sacc data fixture into conftest.py --- tests/conftest.py | 27 +++++++++++ .../statistic/test_cluster_number_counts.py | 45 ++++--------------- tests/test_cluster_data.py | 39 +++------------- 3 files changed, 42 insertions(+), 69 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5bf75437a..d5e93d14e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -128,3 +128,30 @@ def fixture_tools_with_vanilla_cosmology(): result = ModelingTools() result.update(ParamsMap()) result.prepare(pyccl.CosmologyVanillaLCDM()) + + +@pytest.fixture(name="cluster_sacc_data") +def fixture_cluster_sacc_data(): + # pylint: disable=no-member + cc = sacc.standard_types.cluster_counts + # pylint: disable=no-member + mlm = sacc.standard_types.cluster_mean_log_mass + + s = sacc.Sacc() + s.add_tracer("survey", "my_survey", 4000) + s.add_tracer("survey", "not_my_survey", 4000) + s.add_tracer("bin_z", "my_tracer1", 0, 2) + s.add_tracer("bin_z", "my_tracer2", 2, 4) + s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) + s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) + + s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) + s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) + s.add_data_point(cc, ("not_my_survey", "my_tracer1", "my_other_tracer2"), 1) + + s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer2"), 1) + s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) + + return s diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 83618d234..da6d881f1 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -32,33 +32,6 @@ def set_integration_bounds( """Set the limits of integration and extra arguments for the integral""" -@pytest.fixture(name="sacc_data") -def fixture_complicated_sacc_data(): - # pylint: disable=no-member - cc = sacc.standard_types.cluster_counts - # pylint: disable=no-member - mlm = sacc.standard_types.cluster_mean_log_mass - - s = sacc.Sacc() - s.add_tracer("survey", "my_survey", 4000) - s.add_tracer("survey", "not_my_survey", 5000) - s.add_tracer("bin_z", "my_tracer1", 0, 2) - s.add_tracer("bin_z", "my_tracer2", 2, 4) - s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) - s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) - - s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) - s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) - s.add_data_point(cc, ("not_my_survey", "my_tracer1", "my_other_tracer2"), 1) - - s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer2"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) - - return s - - def test_create_binned_number_counts(): integrator = MockIntegrator() bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) @@ -87,7 +60,7 @@ def test_get_data_vector(): assert len(dv) == 0 -def test_read(sacc_data: sacc.Sacc): +def test_read(cluster_sacc_data: sacc.Sacc): integrator = MockIntegrator() bnc = BinnedClusterNumberCounts(False, False, "my_survey", integrator) @@ -95,31 +68,31 @@ def test_read(sacc_data: sacc.Sacc): RuntimeError, match="has read a data vector of length 0; the length must be positive", ): - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 assert len(bnc.bin_limits) == 4 assert len(bnc.data_vector) == 2 assert len(bnc.sacc_indices) == 2 bnc = BinnedClusterNumberCounts(False, True, "my_survey", integrator) - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 assert len(bnc.bin_limits) == 4 assert len(bnc.data_vector) == 4 assert len(bnc.sacc_indices) == 4 bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 assert len(bnc.bin_limits) == 4 assert len(bnc.data_vector) == 6 assert len(bnc.sacc_indices) == 6 -def test_compute_theory_vector(sacc_data: sacc.Sacc): +def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): integrator = MockIntegrator() tools = ModelingTools() @@ -132,19 +105,19 @@ def test_compute_theory_vector(sacc_data: sacc.Sacc): tools.prepare(cosmo, params) bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None assert len(tv) == 4 bnc = BinnedClusterNumberCounts(False, True, "my_survey", integrator) - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None assert len(tv) == 8 bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) - bnc.read(sacc_data) + bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None assert len(tv) == 8 diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index af345dc0e..c5a729c02 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -5,33 +5,6 @@ from firecrown.models.cluster.abundance_data import AbundanceData -@pytest.fixture(name="sacc_data") -def fixture_complicated_sacc_data(): - # pylint: disable=no-member - cc = sacc.standard_types.cluster_counts - # pylint: disable=no-member - mlm = sacc.standard_types.cluster_mean_log_mass - - s = sacc.Sacc() - s.add_tracer("survey", "my_survey", 4000) - s.add_tracer("survey", "not_my_survey", 4000) - s.add_tracer("bin_z", "my_tracer1", 0, 2) - s.add_tracer("bin_z", "my_tracer2", 2, 4) - s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) - s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) - - s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) - s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) - s.add_data_point(cc, ("not_my_survey", "my_tracer1", "my_other_tracer2"), 1) - - s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer2"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) - - return s - - def test_create_abundance_data_no_survey(): with pytest.raises( ValueError, match="The SACC file does not contain the SurveyTracer" @@ -94,8 +67,8 @@ def test_validate_tracers(): ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_counts) -def test_filtered_tracers(sacc_data): - ad = AbundanceData(sacc_data, "my_survey", False, False) +def test_filtered_tracers(cluster_sacc_data): + ad = AbundanceData(cluster_sacc_data, "my_survey", False, False) # pylint: disable=no-member cc = sacc.standard_types.cluster_counts filtered_tracers, survey_mask = ad.get_filtered_tracers(cc) @@ -107,8 +80,8 @@ def test_filtered_tracers(sacc_data): assert (survey_mask == [True, True, False]).all() -def test_get_data_and_indices(sacc_data): - ad = AbundanceData(sacc_data, "my_survey", False, False) +def test_get_data_and_indices(cluster_sacc_data): + ad = AbundanceData(cluster_sacc_data, "my_survey", False, False) # pylint: disable=no-member cc = sacc.standard_types.cluster_counts data, indices = ad.get_data_and_indices(cc) @@ -117,8 +90,8 @@ def test_get_data_and_indices(sacc_data): assert indices == [0, 1] -def test_get_bin_limits(sacc_data): - ad = AbundanceData(sacc_data, "my_survey", False, False) +def test_get_bin_limits(cluster_sacc_data): + ad = AbundanceData(cluster_sacc_data, "my_survey", False, False) # pylint: disable=no-member cc = sacc.standard_types.cluster_counts limits = ad.get_bin_limits(cc) From dcc97cffcccdc0baeb92ca3251f626834f1c0aa5 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 12:05:50 -0800 Subject: [PATCH 52/80] Replacing mock class using built-in unittest.mock. --- tests/test_cluster_integrator_numcosmo.py | 82 ++++++++++++++++------- 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/tests/test_cluster_integrator_numcosmo.py b/tests/test_cluster_integrator_numcosmo.py index d538fdcea..0882bae2a 100644 --- a/tests/test_cluster_integrator_numcosmo.py +++ b/tests/test_cluster_integrator_numcosmo.py @@ -3,6 +3,7 @@ import numpy as np import numpy.typing as npt import pytest +from unittest.mock import Mock from firecrown.models.cluster.integrator.numcosmo_integrator import ( NumCosmoIntegrator, ) @@ -23,22 +24,6 @@ def fixture_cl_abundance(): return cl_abundance -class MockKernel(Kernel): - """A mock implementation of a Kernel used for unit testing.""" - - def distribution( - self, - _mass: npt.NDArray[np.float64], - _z: npt.NDArray[np.float64], - _mass_proxy: npt.NDArray[np.float64], - _z_proxy: npt.NDArray[np.float64], - _mass_proxy_limits: Tuple[float, float], - _z_proxy_limits: Tuple[float, float], - ) -> npt.NDArray[np.float64]: - """The functional form of the distribution or spread of this kernel""" - return np.atleast_1d(1.0) - - def test_numcosmo_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): nci = NumCosmoIntegrator() @@ -68,7 +53,13 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - dd_kernel = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) + dd_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=True, + has_analytic_sln=False, + ) + dd_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -84,7 +75,13 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda } cl_abundance.kernels.clear() - dd_kernel = MockKernel(KernelType.Z_PROXY, is_dirac_delta=True) + dd_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + has_analytic_sln=False, + is_dirac_delta=True, + ) + dd_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -98,7 +95,13 @@ def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbunda KernelType.MASS: 0, KernelType.Z: 1, } - dd_kernel2 = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) + dd_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=False, + is_dirac_delta=True, + ) + dd_kernel2.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(dd_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -124,7 +127,13 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - ig_kernel = MockKernel(KernelType.MASS_PROXY) + ig_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=False, + is_dirac_delta=False, + ) + ig_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -141,7 +150,13 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( } cl_abundance.kernels.clear() - ig_kernel = MockKernel(KernelType.Z_PROXY) + ig_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + has_analytic_sln=False, + is_dirac_delta=False, + ) + ig_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -157,7 +172,13 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( KernelType.Z_PROXY: 2, } - ig_kernel2 = MockKernel(KernelType.MASS_PROXY) + ig_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=False, + is_dirac_delta=False, + ) + ig_kernel2.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(ig_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -185,7 +206,13 @@ def test_numcosmo_set_integration_bounds_analytic_slns( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - a_kernel = MockKernel(KernelType.MASS_PROXY, has_analytic_sln=True) + a_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=True, + is_dirac_delta=False, + ) + a_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(a_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -199,8 +226,13 @@ def test_numcosmo_set_integration_bounds_analytic_slns( KernelType.MASS: 0, KernelType.Z: 1, } - - a_kernel2 = MockKernel(KernelType.Z_PROXY, has_analytic_sln=True) + a_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + has_analytic_sln=True, + is_dirac_delta=False, + ) + a_kernel2.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(a_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): From 2368e0a5d5ab9662f1c5866c2ca2102d854fe5fa Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 12:19:56 -0800 Subject: [PATCH 53/80] Replaced mock kernel with unittest mock in cluster abundance. --- tests/test_cluster_abundance.py | 105 ++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index ae89d95dd..b863d0dee 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,10 +1,9 @@ """Tests for the cluster abundance module.""" -from typing import List, Tuple, Optional import pytest import pyccl import numpy as np -import numpy.typing as npt -from firecrown.parameters import ParamsMap, create +from unittest.mock import Mock +from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import Kernel, KernelType @@ -17,32 +16,6 @@ def fixture_cl_abundance(): return ca -class MockKernel(Kernel): - """A mock implementation of a Kernel used for unit testing.""" - - def __init__( - self, - kernel_type: KernelType, - is_dirac_delta: bool = False, - has_analytic_sln: bool = False, - integral_bounds: Optional[List[Tuple[float, float]]] = None, - ): - super().__init__(kernel_type, is_dirac_delta, has_analytic_sln, integral_bounds) - self.param = create() - - def distribution( - self, - _mass: npt.NDArray[np.float64], - _z: npt.NDArray[np.float64], - _mass_proxy: npt.NDArray[np.float64], - _z_proxy: npt.NDArray[np.float64], - _mass_proxy_limits: Tuple[float, float], - _z_proxy_limits: Tuple[float, float], - ) -> npt.NDArray[np.float64]: - """The functional form of the distribution or spread of this kernel""" - return np.atleast_1d(1.0) - - def test_cluster_abundance_init(cl_abundance: ClusterAbundance): assert cl_abundance is not None assert cl_abundance.cosmo is None @@ -57,21 +30,25 @@ def test_cluster_abundance_init(cl_abundance: ClusterAbundance): def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): - mk = MockKernel(KernelType.MASS_PROXY) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=True, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(mk) assert cl_abundance.cosmo is None - assert mk.param is None # pylint: disable=protected-access assert cl_abundance._hmf_cache == {} - pmap = ParamsMap({"param": 42}) + pmap = ParamsMap({}) cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, pmap) assert cl_abundance.cosmo is not None assert cl_abundance.cosmo == cosmo - assert mk.param == 42 # pylint: disable=protected-access assert cl_abundance._hmf_cache == {} @@ -94,7 +71,14 @@ def test_cluster_sky_area(cl_abundance: ClusterAbundance): def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.kernels) == 0 - cl_abundance.add_kernel(MockKernel(KernelType.MASS)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) assert len(cl_abundance.kernels) == 1 assert isinstance(cl_abundance.kernels[0], Kernel) assert cl_abundance.kernels[0].kernel_type == KernelType.MASS @@ -103,12 +87,25 @@ def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.dirac_delta_kernels) == 0 assert len(cl_abundance.integrable_kernels) == 1 - cl_abundance.add_kernel(MockKernel(KernelType.MASS, True)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=True, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) assert len(cl_abundance.analytic_kernels) == 0 assert len(cl_abundance.dirac_delta_kernels) == 1 assert len(cl_abundance.integrable_kernels) == 1 - cl_abundance.add_kernel(MockKernel(KernelType.MASS, False, True)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=True, + ) + cl_abundance.add_kernel(mk) assert len(cl_abundance.analytic_kernels) == 1 assert len(cl_abundance.dirac_delta_kernels) == 1 assert len(cl_abundance.integrable_kernels) == 1 @@ -139,7 +136,14 @@ def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): def test_abundance_get_integrand(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.MASS)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) @@ -158,7 +162,14 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.MASS)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) @@ -177,7 +188,14 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.MASS)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) @@ -196,7 +214,14 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - cl_abundance.add_kernel(MockKernel(KernelType.MASS)) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) From ab08fe24a9112b0962159e51c53f9f2f04c4f5cc Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 12:24:28 -0800 Subject: [PATCH 54/80] Removing mock kernel in favor of unittest.mock for integrators. --- tests/test_cluster_integrator_scipy.py | 82 ++++++++++++++++++-------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/tests/test_cluster_integrator_scipy.py b/tests/test_cluster_integrator_scipy.py index 654e7522e..e411a00ae 100644 --- a/tests/test_cluster_integrator_scipy.py +++ b/tests/test_cluster_integrator_scipy.py @@ -1,5 +1,6 @@ """Tests for the scipy integrator module.""" from typing import Tuple +from unittest.mock import Mock import numpy as np import numpy.typing as npt import pytest @@ -23,22 +24,6 @@ def fixture_cl_abundance(): return cl_abundance -class MockKernel(Kernel): - """A mock implementation of a Kernel used for unit testing.""" - - def distribution( - self, - _mass: npt.NDArray[np.float64], - _z: npt.NDArray[np.float64], - _mass_proxy: npt.NDArray[np.float64], - _z_proxy: npt.NDArray[np.float64], - _mass_proxy_limits: Tuple[float, float], - _z_proxy_limits: Tuple[float, float], - ) -> npt.NDArray[np.float64]: - """The functional form of the distribution or spread of this kernel""" - return np.atleast_1d(1.0) - - def test_scipy_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): spi = ScipyIntegrator() @@ -68,7 +53,13 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - dd_kernel = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) + dd_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=True, + has_analytic_sln=False, + ) + dd_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -84,7 +75,13 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance } cl_abundance.kernels.clear() - dd_kernel = MockKernel(KernelType.Z_PROXY, is_dirac_delta=True) + dd_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + is_dirac_delta=True, + has_analytic_sln=False, + ) + dd_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(dd_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -98,7 +95,14 @@ def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance KernelType.MASS: 0, KernelType.Z: 1, } - dd_kernel2 = MockKernel(KernelType.MASS_PROXY, is_dirac_delta=True) + + dd_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=True, + has_analytic_sln=False, + ) + dd_kernel2.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(dd_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -124,7 +128,13 @@ def test_scipy_set_integration_bounds_integrable_kernels( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - ig_kernel = MockKernel(KernelType.MASS_PROXY) + ig_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=False, + has_analytic_sln=False, + ) + ig_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -141,7 +151,13 @@ def test_scipy_set_integration_bounds_integrable_kernels( } cl_abundance.kernels.clear() - ig_kernel = MockKernel(KernelType.Z_PROXY) + ig_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + is_dirac_delta=False, + has_analytic_sln=False, + ) + ig_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(ig_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -157,7 +173,13 @@ def test_scipy_set_integration_bounds_integrable_kernels( KernelType.Z_PROXY: 2, } - ig_kernel2 = MockKernel(KernelType.MASS_PROXY) + ig_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=False, + has_analytic_sln=False, + ) + ig_kernel2.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(ig_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -185,7 +207,13 @@ def test_scipy_set_integration_bounds_analytic_slns( z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) - a_kernel = MockKernel(KernelType.MASS_PROXY, has_analytic_sln=True) + a_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=False, + has_analytic_sln=True, + ) + a_kernel.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(a_kernel) for z_limits, mass_limits in zip(z_bins, m_bins): @@ -200,7 +228,13 @@ def test_scipy_set_integration_bounds_analytic_slns( KernelType.Z: 1, } - a_kernel2 = MockKernel(KernelType.Z_PROXY, has_analytic_sln=True) + a_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + is_dirac_delta=False, + has_analytic_sln=True, + ) + a_kernel2.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(a_kernel2) for z_limits, mass_limits in zip(z_bins, m_bins): From 087e2e9bbfa8651dd67314c122e1a7e8eb2220b0 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 12:46:58 -0800 Subject: [PATCH 55/80] Unifying tests for numcosmo and scipy. --- tests/conftest.py | 16 +- tests/test_cluster_abundance.py | 2 +- tests/test_cluster_integrator_numcosmo.py | 277 --------------------- tests/test_cluster_integrator_scipy.py | 279 --------------------- tests/test_cluster_integrators.py | 284 ++++++++++++++++++++++ 5 files changed, 300 insertions(+), 558 deletions(-) delete mode 100644 tests/test_cluster_integrator_numcosmo.py delete mode 100644 tests/test_cluster_integrator_scipy.py create mode 100644 tests/test_cluster_integrators.py diff --git a/tests/conftest.py b/tests/conftest.py index d5e93d14e..b052edd75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ from firecrown.parameters import ParamsMap from firecrown.connector.mapping import MappingCosmoSIS, mapping_builder from firecrown.modeling_tools import ModelingTools +from firecrown.models.cluster.abundance import ClusterAbundance def pytest_addoption(parser): @@ -131,7 +132,7 @@ def fixture_tools_with_vanilla_cosmology(): @pytest.fixture(name="cluster_sacc_data") -def fixture_cluster_sacc_data(): +def fixture_cluster_sacc_data() -> sacc.Sacc: # pylint: disable=no-member cc = sacc.standard_types.cluster_counts # pylint: disable=no-member @@ -155,3 +156,16 @@ def fixture_cluster_sacc_data(): s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) return s + + +@pytest.fixture(name="empty_cluster_abundance") +def fixture_empty_cluster_abundance() -> ClusterAbundance: + cl_abundance = ClusterAbundance( + min_z=0, + max_z=2, + min_mass=13, + max_mass=17, + sky_area=100, + halo_mass_function=None, + ) + return cl_abundance diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index b863d0dee..740c40108 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -1,8 +1,8 @@ """Tests for the cluster abundance module.""" +from unittest.mock import Mock import pytest import pyccl import numpy as np -from unittest.mock import Mock from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import Kernel, KernelType diff --git a/tests/test_cluster_integrator_numcosmo.py b/tests/test_cluster_integrator_numcosmo.py deleted file mode 100644 index 0882bae2a..000000000 --- a/tests/test_cluster_integrator_numcosmo.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Tests for the numcosmo integrator module.""" -from typing import Tuple -import numpy as np -import numpy.typing as npt -import pytest -from unittest.mock import Mock -from firecrown.models.cluster.integrator.numcosmo_integrator import ( - NumCosmoIntegrator, -) -from firecrown.models.cluster.kernel import KernelType, Kernel -from firecrown.models.cluster.abundance import ClusterAbundance - - -@pytest.fixture(name="cl_abundance") -def fixture_cl_abundance(): - cl_abundance = ClusterAbundance( - min_z=0, - max_z=2, - min_mass=13, - max_mass=17, - sky_area=100, - halo_mass_function=None, - ) - return cl_abundance - - -def test_numcosmo_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): - nci = NumCosmoIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 2 - assert nci.integral_bounds == [(13, 17), (0, 2)] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - -def test_numcosmo_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): - nci = NumCosmoIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - dd_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - is_dirac_delta=True, - has_analytic_sln=False, - ) - dd_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(dd_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 2 - assert nci.integral_bounds == [mass_limits, (0, 2)] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - cl_abundance.kernels.clear() - dd_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.Z_PROXY, - has_analytic_sln=False, - is_dirac_delta=True, - ) - dd_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(dd_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 2 - assert nci.integral_bounds == [(13, 17), z_limits] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - dd_kernel2 = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - has_analytic_sln=False, - is_dirac_delta=True, - ) - dd_kernel2.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(dd_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 2 - assert nci.integral_bounds == [mass_limits, z_limits] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - -def test_numcosmo_set_integration_bounds_integrable_kernels( - cl_abundance: ClusterAbundance, -): - nci = NumCosmoIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - ig_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - has_analytic_sln=False, - is_dirac_delta=False, - ) - ig_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(ig_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 3 - assert nci.integral_bounds == [(13, 17), (0, 2), mass_limits] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - KernelType.MASS_PROXY: 2, - } - - cl_abundance.kernels.clear() - ig_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.Z_PROXY, - has_analytic_sln=False, - is_dirac_delta=False, - ) - ig_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(ig_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 3 - assert nci.integral_bounds == [(13, 17), (0, 2), z_limits] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - KernelType.Z_PROXY: 2, - } - - ig_kernel2 = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - has_analytic_sln=False, - is_dirac_delta=False, - ) - ig_kernel2.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(ig_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 4 - assert nci.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - KernelType.Z_PROXY: 2, - KernelType.MASS_PROXY: 3, - } - - -def test_numcosmo_set_integration_bounds_analytic_slns( - cl_abundance: ClusterAbundance, -): - nci = NumCosmoIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - a_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - has_analytic_sln=True, - is_dirac_delta=False, - ) - a_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(a_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 2 - assert nci.integral_bounds == [(13, 17), (0, 2)] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - a_kernel2 = Mock( - spec=Kernel, - kernel_type=KernelType.Z_PROXY, - has_analytic_sln=True, - is_dirac_delta=False, - ) - a_kernel2.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(a_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - nci.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert nci.mass_proxy_limits == mass_limits - assert nci.z_proxy_limits == z_limits - assert len(nci.integral_bounds) == 2 - assert nci.integral_bounds == [(13, 17), (0, 2)] - assert nci.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - -def test_numcosmo_integrator_integrate(cl_abundance: ClusterAbundance): - nci = NumCosmoIntegrator() - cl_abundance.min_mass = 0 - cl_abundance.max_mass = 1 - cl_abundance.min_z = 0 - cl_abundance.max_z = 1 - - def integrand( - a: npt.NDArray[np.float64], - b: npt.NDArray[np.float64], - _c: npt.NDArray[np.float64], - _d: npt.NDArray[np.float64], - _e: Tuple[float, float], - _f: Tuple[float, float], - ): - # xy - result = a * b - return result - - nci.set_integration_bounds( - cl_abundance, - (0, 1), - (0, 1), - ) - result = nci.integrate(integrand) - # \int_0^1 \int_0^1 xy dx dy = 1/4 - assert result == 0.25 diff --git a/tests/test_cluster_integrator_scipy.py b/tests/test_cluster_integrator_scipy.py deleted file mode 100644 index e411a00ae..000000000 --- a/tests/test_cluster_integrator_scipy.py +++ /dev/null @@ -1,279 +0,0 @@ -"""Tests for the scipy integrator module.""" -from typing import Tuple -from unittest.mock import Mock -import numpy as np -import numpy.typing as npt -import pytest -from firecrown.models.cluster.integrator.scipy_integrator import ( - ScipyIntegrator, -) -from firecrown.models.cluster.kernel import KernelType, Kernel -from firecrown.models.cluster.abundance import ClusterAbundance - - -@pytest.fixture(name="cl_abundance") -def fixture_cl_abundance(): - cl_abundance = ClusterAbundance( - min_z=0, - max_z=2, - min_mass=13, - max_mass=17, - sky_area=100, - halo_mass_function=None, - ) - return cl_abundance - - -def test_scipy_set_integration_bounds_no_kernels(cl_abundance: ClusterAbundance): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 2 - assert spi.integral_bounds == [(13, 17), (0, 2)] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - -def test_scipy_set_integration_bounds_dirac_delta(cl_abundance: ClusterAbundance): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - dd_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - is_dirac_delta=True, - has_analytic_sln=False, - ) - dd_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(dd_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 2 - assert spi.integral_bounds == [mass_limits, (0, 2)] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - cl_abundance.kernels.clear() - dd_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.Z_PROXY, - is_dirac_delta=True, - has_analytic_sln=False, - ) - dd_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(dd_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 2 - assert spi.integral_bounds == [(13, 17), z_limits] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - dd_kernel2 = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - is_dirac_delta=True, - has_analytic_sln=False, - ) - dd_kernel2.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(dd_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 2 - assert spi.integral_bounds == [mass_limits, z_limits] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - -def test_scipy_set_integration_bounds_integrable_kernels( - cl_abundance: ClusterAbundance, -): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - ig_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - is_dirac_delta=False, - has_analytic_sln=False, - ) - ig_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(ig_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 3 - assert spi.integral_bounds == [(13, 17), (0, 2), mass_limits] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - KernelType.MASS_PROXY: 2, - } - - cl_abundance.kernels.clear() - ig_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.Z_PROXY, - is_dirac_delta=False, - has_analytic_sln=False, - ) - ig_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(ig_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 3 - assert spi.integral_bounds == [(13, 17), (0, 2), z_limits] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - KernelType.Z_PROXY: 2, - } - - ig_kernel2 = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - is_dirac_delta=False, - has_analytic_sln=False, - ) - ig_kernel2.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(ig_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 4 - assert spi.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - KernelType.Z_PROXY: 2, - KernelType.MASS_PROXY: 3, - } - - -def test_scipy_set_integration_bounds_analytic_slns( - cl_abundance: ClusterAbundance, -): - spi = ScipyIntegrator() - - z_array = np.linspace(0, 2, 10) - m_array = np.linspace(13, 17, 10) - z_bins = list(zip(z_array[:-1], z_array[1:])) - m_bins = list(zip(m_array[:-1], m_array[1:])) - - a_kernel = Mock( - spec=Kernel, - kernel_type=KernelType.MASS_PROXY, - is_dirac_delta=False, - has_analytic_sln=True, - ) - a_kernel.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(a_kernel) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 2 - assert spi.integral_bounds == [(13, 17), (0, 2)] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - a_kernel2 = Mock( - spec=Kernel, - kernel_type=KernelType.Z_PROXY, - is_dirac_delta=False, - has_analytic_sln=True, - ) - a_kernel2.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(a_kernel2) - - for z_limits, mass_limits in zip(z_bins, m_bins): - spi.set_integration_bounds(cl_abundance, z_limits, mass_limits) - - assert spi.mass_proxy_limits == mass_limits - assert spi.z_proxy_limits == z_limits - assert len(spi.integral_bounds) == 2 - assert spi.integral_bounds == [(13, 17), (0, 2)] - assert spi.integral_args_lkp == { - KernelType.MASS: 0, - KernelType.Z: 1, - } - - -def test_scipy_integrator_integrate(cl_abundance: ClusterAbundance): - spi = ScipyIntegrator() - cl_abundance.min_mass = 0 - cl_abundance.max_mass = 1 - cl_abundance.min_z = 0 - cl_abundance.max_z = 1 - - def integrand( - a: npt.NDArray[np.float64], - b: npt.NDArray[np.float64], - _c: npt.NDArray[np.float64], - _d: npt.NDArray[np.float64], - _e: Tuple[float, float], - _f: Tuple[float, float], - ): - # xy - result = a * b - return result - - spi.set_integration_bounds( - cl_abundance, - (0, 1), - (0, 1), - ) - result = spi.integrate(integrand) - # \int_0^1 \int_0^1 xy dx dy = 1/4 - assert result == pytest.approx(0.25, rel=1e-15, abs=0) diff --git a/tests/test_cluster_integrators.py b/tests/test_cluster_integrators.py new file mode 100644 index 000000000..28e3fe0af --- /dev/null +++ b/tests/test_cluster_integrators.py @@ -0,0 +1,284 @@ +"""Tests for the numcosmo integrator module.""" +from typing import Tuple +from unittest.mock import Mock +import numpy as np +import numpy.typing as npt +import pytest +from firecrown.models.cluster.integrator.integrator import Integrator +from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator +from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator +from firecrown.models.cluster.kernel import KernelType, Kernel +from firecrown.models.cluster.abundance import ClusterAbundance + + +@pytest.fixture(name="integrator", params=[ScipyIntegrator, NumCosmoIntegrator]) +def fixture_integrator(request) -> Integrator: + return request.param() + + +def test_numcosmo_set_integration_bounds_no_kernels( + empty_cluster_abundance: ClusterAbundance, integrator: Integrator +): + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 2 + assert integrator.integral_bounds == [(13, 17), (0, 2)] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + } + + +def test_numcosmo_set_integration_bounds_dirac_delta( + empty_cluster_abundance: ClusterAbundance, integrator: Integrator +): + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + dd_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + is_dirac_delta=True, + has_analytic_sln=False, + ) + dd_kernel.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 2 + assert integrator.integral_bounds == [mass_limits, (0, 2)] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + } + + empty_cluster_abundance.kernels.clear() + dd_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + has_analytic_sln=False, + is_dirac_delta=True, + ) + dd_kernel.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(dd_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 2 + assert integrator.integral_bounds == [(13, 17), z_limits] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + } + dd_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=False, + is_dirac_delta=True, + ) + dd_kernel2.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(dd_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 2 + assert integrator.integral_bounds == [mass_limits, z_limits] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + } + + +def test_numcosmo_set_integration_bounds_integrable_kernels( + empty_cluster_abundance: ClusterAbundance, integrator: Integrator +): + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + ig_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=False, + is_dirac_delta=False, + ) + ig_kernel.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 3 + assert integrator.integral_bounds == [(13, 17), (0, 2), mass_limits] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.MASS_PROXY: 2, + } + + empty_cluster_abundance.kernels.clear() + ig_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + has_analytic_sln=False, + is_dirac_delta=False, + ) + ig_kernel.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(ig_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 3 + assert integrator.integral_bounds == [(13, 17), (0, 2), z_limits] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.Z_PROXY: 2, + } + + ig_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=False, + is_dirac_delta=False, + ) + ig_kernel2.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(ig_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 4 + assert integrator.integral_bounds == [(13, 17), (0, 2), z_limits, mass_limits] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + KernelType.Z_PROXY: 2, + KernelType.MASS_PROXY: 3, + } + + +def test_numcosmo_set_integration_bounds_analytic_slns( + empty_cluster_abundance: ClusterAbundance, integrator: Integrator +): + z_array = np.linspace(0, 2, 10) + m_array = np.linspace(13, 17, 10) + z_bins = list(zip(z_array[:-1], z_array[1:])) + m_bins = list(zip(m_array[:-1], m_array[1:])) + + a_kernel = Mock( + spec=Kernel, + kernel_type=KernelType.MASS_PROXY, + has_analytic_sln=True, + is_dirac_delta=False, + ) + a_kernel.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(a_kernel) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 2 + assert integrator.integral_bounds == [(13, 17), (0, 2)] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + } + a_kernel2 = Mock( + spec=Kernel, + kernel_type=KernelType.Z_PROXY, + has_analytic_sln=True, + is_dirac_delta=False, + ) + a_kernel2.distribution.return_value = np.atleast_1d(1.0) + empty_cluster_abundance.add_kernel(a_kernel2) + + for z_limits, mass_limits in zip(z_bins, m_bins): + integrator.set_integration_bounds( + empty_cluster_abundance, z_limits, mass_limits + ) + + assert integrator.mass_proxy_limits == mass_limits + assert integrator.z_proxy_limits == z_limits + assert len(integrator.integral_bounds) == 2 + assert integrator.integral_bounds == [(13, 17), (0, 2)] + assert integrator.integral_args_lkp == { + KernelType.MASS: 0, + KernelType.Z: 1, + } + + +def test_numcosmo_integrator_integrate( + empty_cluster_abundance: ClusterAbundance, integrator: Integrator +): + empty_cluster_abundance.min_mass = 0 + empty_cluster_abundance.max_mass = 1 + empty_cluster_abundance.min_z = 0 + empty_cluster_abundance.max_z = 1 + + def integrand( + a: npt.NDArray[np.float64], + b: npt.NDArray[np.float64], + _c: npt.NDArray[np.float64], + _d: npt.NDArray[np.float64], + _e: Tuple[float, float], + _f: Tuple[float, float], + ): + # xy + result = a * b + return result + + integrator.set_integration_bounds( + empty_cluster_abundance, + (0, 1), + (0, 1), + ) + result = integrator.integrate(integrand) + # \int_0^1 \int_0^1 xy dx dy = 1/4 + assert result == pytest.approx(0.25, rel=1e-15, abs=0) From 0cb7767436a2806ae1b7dba7042d6ff711313cd5 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 12:53:21 -0800 Subject: [PATCH 56/80] Replace variable with inline call. --- tests/test_cluster_kernels.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index aec37bcd1..03f6f2a62 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -60,16 +60,14 @@ def test_create_purity_kernel(): def test_spec_z_distribution(): srk = SpectroscopicRedshift() - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) - assert ( srk.distribution( - mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + mass=np.linspace(13, 17, 5), + z=np.linspace(0, 1, 5), + mass_proxy=np.linspace(0, 5, 5), + z_proxy=np.linspace(0, 1, 5), + mass_proxy_limits=(0, 5), + z_proxy_limits=(0, 1), ) == 1.0 ) @@ -77,16 +75,15 @@ def test_spec_z_distribution(): def test_true_mass_distribution(): tmk = TrueMass() - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) assert ( tmk.distribution( - mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + mass=np.linspace(13, 17, 5), + z=np.linspace(0, 1, 5), + mass_proxy=np.linspace(0, 5, 5), + z_proxy=np.linspace(0, 1, 5), + mass_proxy_limits=(0, 5), + z_proxy_limits=(0, 1), ) == 1.0 ) From 3ccdb65fe8cae0f4f3b9ebc259d3c990b6c9fdf5 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 12:58:40 -0800 Subject: [PATCH 57/80] Fixing error in cluster kernel tests. --- tests/test_cluster_kernels.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 03f6f2a62..6f30fbb50 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -62,12 +62,12 @@ def test_spec_z_distribution(): assert ( srk.distribution( - mass=np.linspace(13, 17, 5), - z=np.linspace(0, 1, 5), - mass_proxy=np.linspace(0, 5, 5), - z_proxy=np.linspace(0, 1, 5), - mass_proxy_limits=(0, 5), - z_proxy_limits=(0, 1), + _mass=np.linspace(13, 17, 5), + _z=np.linspace(0, 1, 5), + _mass_proxy=np.linspace(0, 5, 5), + _z_proxy=np.linspace(0, 1, 5), + _mass_proxy_limits=(0, 5), + _z_proxy_limits=(0, 1), ) == 1.0 ) @@ -78,12 +78,12 @@ def test_true_mass_distribution(): assert ( tmk.distribution( - mass=np.linspace(13, 17, 5), - z=np.linspace(0, 1, 5), - mass_proxy=np.linspace(0, 5, 5), - z_proxy=np.linspace(0, 1, 5), - mass_proxy_limits=(0, 5), - z_proxy_limits=(0, 1), + _mass=np.linspace(13, 17, 5), + _z=np.linspace(0, 1, 5), + _mass_proxy=np.linspace(0, 5, 5), + _z_proxy=np.linspace(0, 1, 5), + _mass_proxy_limits=(0, 5), + _z_proxy_limits=(0, 1), ) == 1.0 ) From 2917a77bd2726e9e17a4adcecf90ed535607dff9 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 13:34:48 -0800 Subject: [PATCH 58/80] Updating unit test to capture error found on CI. --- .../statistic/test_cluster_number_counts.py | 37 ++++++------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index da6d881f1..7390ee81e 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -1,5 +1,5 @@ """Tests for the binned cluster number counts module.""" -from typing import Tuple +from unittest.mock import Mock import sacc import pytest import pyccl @@ -7,33 +7,15 @@ from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic from firecrown.modeling_tools import ModelingTools from firecrown.parameters import ParamsMap -from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand +from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( BinnedClusterNumberCounts, ) -class MockIntegrator(Integrator): - """A mock integrator used by the cluster number counts statistic for testing""" - - def integrate( - self, - integrand: AbundanceIntegrand, - ) -> float: - """Integrate the integrand over the bounds and include extra_args to integral""" - return 1.0 - - def set_integration_bounds( - self, - cl_abundance: ClusterAbundance, - z_proxy_limits: Tuple[float, float], - mass_proxy_limits: Tuple[float, float], - ) -> None: - """Set the limits of integration and extra arguments for the integral""" - - def test_create_binned_number_counts(): - integrator = MockIntegrator() + integrator = Mock(spec=Integrator) + integrator.integrate.return_value = 1.0 bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) assert bnc is not None assert bnc.use_cluster_counts is False @@ -53,7 +35,8 @@ def test_create_binned_number_counts(): def test_get_data_vector(): - integrator = MockIntegrator() + integrator = Mock(spec=Integrator) + integrator.integrate.return_value = 1.0 bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) dv = bnc.get_data_vector() assert dv is not None @@ -61,7 +44,8 @@ def test_get_data_vector(): def test_read(cluster_sacc_data: sacc.Sacc): - integrator = MockIntegrator() + integrator = Mock(spec=Integrator) + integrator.integrate.return_value = 1.0 bnc = BinnedClusterNumberCounts(False, False, "my_survey", integrator) with pytest.raises( @@ -93,7 +77,8 @@ def test_read(cluster_sacc_data: sacc.Sacc): def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): - integrator = MockIntegrator() + integrator = Mock(spec=Integrator) + integrator.integrate.return_value = 1.0 tools = ModelingTools() hmf = pyccl.halos.MassFuncBocquet16() @@ -114,7 +99,7 @@ def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None - assert len(tv) == 8 + assert len(tv) == 4 bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) bnc.read(cluster_sacc_data) From e61fcee2000b98eae5d6302121d7b2a7090c8df7 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 13:50:50 -0800 Subject: [PATCH 59/80] Fixed test and extracted a method to replace some confusing conditonal logic. --- .../statistic/binned_cluster_number_counts.py | 63 ++++++++++++------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index a6eebbd1e..040e5cd26 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -87,34 +87,53 @@ def get_data_vector(self) -> DataVector: def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: assert tools.cluster_abundance is not None - theory_vector_list = [] + theory_vector_list: List[float] = [] cluster_counts = [] - cluster_masses = [] - if self.use_cluster_counts or self.use_mean_log_mass: - for z_proxy_limits, mass_proxy_limits in self.bin_limits: - self.integrator.set_integration_bounds( - tools.cluster_abundance, z_proxy_limits, mass_proxy_limits - ) + if not self.use_cluster_counts and not self.use_mean_log_mass: + return TheoryVector.from_list(theory_vector_list) + + cluster_counts = self.get_binned_cluster_counts(tools) - integrand = tools.cluster_abundance.get_integrand() - counts = self.integrator.integrate(integrand) - cluster_counts.append(counts) + if self.use_cluster_counts: theory_vector_list += cluster_counts if self.use_mean_log_mass: - for (z_proxy_limits, mass_proxy_limits), counts in zip( - self.bin_limits, cluster_counts - ): - integrand = tools.cluster_abundance.get_integrand(avg_mass=True) - self.integrator.set_integration_bounds( - tools.cluster_abundance, z_proxy_limits, mass_proxy_limits - ) + theory_vector_list += self.get_binned_cluster_masses(tools, cluster_counts) - total_mass = self.integrator.integrate(integrand) - mean_mass = total_mass / counts - cluster_masses.append(mean_mass) + return TheoryVector.from_list(theory_vector_list) - theory_vector_list += cluster_masses + def get_binned_cluster_masses( + self, tools: ModelingTools, cluster_counts: List[float] + ) -> List[float]: + assert tools.cluster_abundance is not None - return TheoryVector.from_list(theory_vector_list) + cluster_masses = [] + for (z_proxy_limits, mass_proxy_limits), counts in zip( + self.bin_limits, cluster_counts + ): + integrand = tools.cluster_abundance.get_integrand(avg_mass=True) + self.integrator.set_integration_bounds( + tools.cluster_abundance, z_proxy_limits, mass_proxy_limits + ) + + total_mass = self.integrator.integrate(integrand) + mean_mass = total_mass / counts + cluster_masses.append(mean_mass) + + return cluster_masses + + def get_binned_cluster_counts(self, tools: ModelingTools) -> List[float]: + assert tools.cluster_abundance is not None + + cluster_counts = [] + for z_proxy_limits, mass_proxy_limits in self.bin_limits: + self.integrator.set_integration_bounds( + tools.cluster_abundance, z_proxy_limits, mass_proxy_limits + ) + + integrand = tools.cluster_abundance.get_integrand() + counts = self.integrator.integrate(integrand) + cluster_counts.append(counts) + + return cluster_counts From 50079fc504a39bfe3b882d2d6d75965148b15ae2 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 10 Nov 2023 13:55:02 -0800 Subject: [PATCH 60/80] Adding docstrings to my new public methods. --- .../statistic/binned_cluster_number_counts.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 040e5cd26..5408b2594 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -106,6 +106,11 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: def get_binned_cluster_masses( self, tools: ModelingTools, cluster_counts: List[float] ) -> List[float]: + """Computes the mean mass of clusters in each bin + + Using the data from the sacc file, this function evaluates the likelihood for + a single point of the parameter space, and returns the predicted mean mass of + the clusters in each bin.""" assert tools.cluster_abundance is not None cluster_masses = [] @@ -124,6 +129,11 @@ def get_binned_cluster_masses( return cluster_masses def get_binned_cluster_counts(self, tools: ModelingTools) -> List[float]: + """Computes the number of clusters in each bin + + Using the data from the sacc file, this function evaluates the likelihood for + a single point of the parameter space, and returns the predicted number of + clusters in each bin.""" assert tools.cluster_abundance is not None cluster_counts = [] From 77b73f6bc291f8350be4a78cc3d152ebb96e77b1 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 28 Nov 2023 11:17:45 -0800 Subject: [PATCH 61/80] Moving sky area from a constructor parameter to an argument to the cluster abundance Adding some tests to capture some edge cases exposed when moving sky_area. --- .../cluster_redshift_richness.py | 11 ++--- firecrown/models/cluster/abundance.py | 36 ++++++++--------- .../models/cluster/integrator/integrator.py | 17 +++++++- .../cluster/integrator/numcosmo_integrator.py | 6 +++ .../cluster/integrator/scipy_integrator.py | 10 ++++- pytest.ini | 2 +- tests/conftest.py | 1 - .../statistic/test_cluster_number_counts.py | 2 +- tests/test_cluster_abundance.py | 40 +++++++++++-------- tests/test_cluster_integrators.py | 33 +++++++++------ 10 files changed, 94 insertions(+), 64 deletions(-) diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index 332682415..b46e6509d 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -21,13 +21,11 @@ from typing import Tuple -def get_cluster_abundance(sky_area: float) -> ClusterAbundance: +def get_cluster_abundance() -> ClusterAbundance: hmf = ccl.halos.MassFuncBocquet16() min_mass, max_mass = 13.0, 16.0 min_z, max_z = 0.2, 0.8 - cluster_abundance = ClusterAbundance( - min_mass, max_mass, min_z, max_z, hmf, sky_area - ) + cluster_abundance = ClusterAbundance(min_mass, max_mass, min_z, max_z, hmf) # Create and add the kernels you want in your cluster abundance pivot_mass, pivot_redshift = 14.625862906, 0.6 @@ -69,10 +67,7 @@ def build_likelihood( sacc_data = sacc.Sacc.load_fits(os.path.join(sacc_path, sacc_file_nm)) likelihood.read(sacc_data) - sacc_adapter = AbundanceData( - sacc_data, survey_name, use_cluster_counts, use_mean_log_mass - ) - cluster_abundance = get_cluster_abundance(sacc_adapter.survey_tracer.sky_area) + cluster_abundance = get_cluster_abundance() modeling_tools = ModelingTools(cluster_abundance=cluster_abundance) return likelihood, modeling_tools diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 08c019631..bf8da19b5 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -18,6 +18,7 @@ [ npt.NDArray[np.float64], npt.NDArray[np.float64], + float, npt.NDArray[np.float64], npt.NDArray[np.float64], Tuple[float, float], @@ -36,19 +37,6 @@ class ClusterAbundance: abundance integrand. """ - @property - def sky_area(self) -> float: - """The sky area in square degrees - - The cluster number count prediction will be different depending on - the area of the sky we are considering. This value controls that. - """ - return self.sky_area_rad * (180.0 / np.pi) ** 2 - - @sky_area.setter - def sky_area(self, sky_area: float) -> None: - self.sky_area_rad = sky_area * (np.pi / 180.0) ** 2 - @property def cosmo(self) -> Cosmology: """The cosmology used to predict the cluster number count.""" @@ -78,7 +66,6 @@ def __init__( min_z: float, max_z: float, halo_mass_function: pyccl.halos.MassFunc, - sky_area: float, ) -> None: self.kernels: List[Kernel] = [] self.halo_mass_function = halo_mass_function @@ -86,7 +73,6 @@ def __init__( self.max_mass = max_mass self.min_z = min_z self.max_z = max_z - self.sky_area = sky_area self._hmf_cache: Dict[Tuple[float, float], float] = {} self._cosmo: Cosmology = None @@ -106,8 +92,12 @@ def update_ingredients( for kernel in self.kernels: kernel.update(params) - def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: - """The differential comoving volume at redshift z.""" + def comoving_volume( + self, z: npt.NDArray[np.float64], sky_area: float = 0 + ) -> npt.NDArray[np.float64]: + """The differential comoving volume given area sky_area at redshift z. + + :param sky_area: The area of the survey on the sky in square degrees.""" scale_factor = 1.0 / (1.0 + z) angular_diam_dist = bkg.angular_diameter_distance(self.cosmo, scale_factor) h_over_h0 = bkg.h_over_h0(self.cosmo, scale_factor) @@ -119,10 +109,15 @@ def comoving_volume(self, z: npt.NDArray[np.float64]) -> npt.NDArray[np.float64] / (self.cosmo["h"] * h_over_h0) ) assert isinstance(dV, np.ndarray) - return dV * self.sky_area_rad + + sky_area_rad = sky_area * (np.pi / 180.0) ** 2 + + return dV * sky_area_rad def mass_function( - self, mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64] + self, + mass: npt.NDArray[np.float64], + z: npt.NDArray[np.float64], ) -> npt.NDArray[np.float64]: """The mass function at z and mass.""" scale_factor = 1.0 / (1.0 + z) @@ -145,12 +140,13 @@ def get_integrand( def integrand( mass: npt.NDArray[np.float64], z: npt.NDArray[np.float64], + sky_area: float, mass_proxy: npt.NDArray[np.float64], z_proxy: npt.NDArray[np.float64], mass_proxy_limits: Tuple[float, float], z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: - integrand = self.comoving_volume(z) * self.mass_function(mass, z) + integrand = self.comoving_volume(z, sky_area) * self.mass_function(mass, z) if avg_mass: integrand *= mass diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 539bc8330..40292c40b 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -3,8 +3,9 @@ This module holds the classes that define the interface required to integrate an assembled cluster abundance. """ +import inspect from abc import ABC, abstractmethod -from typing import Tuple, Dict +from typing import Tuple, Dict, Callable, get_args from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand from firecrown.models.cluster.kernel import KernelType @@ -26,6 +27,7 @@ def integrate( def set_integration_bounds( self, cl_abundance: ClusterAbundance, + sky_area: float, z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: @@ -40,3 +42,16 @@ def _default_integral_args(self) -> Dict[KernelType, int]: lkp[KernelType.MASS] = 0 lkp[KernelType.Z] = 1 return lkp + + def _validate_integrand(self, integrand: AbundanceIntegrand) -> None: + expected_args, expected_return = get_args(AbundanceIntegrand) + + signature = inspect.signature(integrand) + params = signature.parameters.values() + param_types = [param.annotation for param in params] + + assert len(params) == len(expected_args) + + assert param_types == list(expected_args) + + assert signature.return_annotation == expected_return diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index dffb4adb5..7b3335493 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -26,11 +26,14 @@ def __init__( self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + self.sky_area: float = 360**2 def _integral_wrapper( self, integrand: AbundanceIntegrand, ) -> Callable[[npt.NDArray], Sequence[float]]: + self._validate_integrand(integrand) + # mypy strict issue: npt.NDArray[npt.NDArray[np.float64]] not supported def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: default = np.ones_like(int_args[0]) * -1.0 @@ -43,6 +46,7 @@ def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: return_val = integrand( mass, z, + self.sky_area, mass_proxy, z_proxy, self.mass_proxy_limits, @@ -56,6 +60,7 @@ def ncm_integrand(int_args: npt.NDArray) -> Sequence[float]: def set_integration_bounds( self, cl_abundance: ClusterAbundance, + sky_area: float, z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: @@ -68,6 +73,7 @@ def set_integration_bounds( self.mass_proxy_limits = mass_proxy_limits self.z_proxy_limits = z_proxy_limits + self.sky_area = sky_area for kernel in cl_abundance.dirac_delta_kernels: if kernel.kernel_type == KernelType.Z_PROXY: diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index 1fdbbbc9d..2120b762a 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -26,11 +26,14 @@ def __init__( self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + self.sky_area: float = 360**2 - def _integral_wrapper( + def integral_wrapper( self, integrand: AbundanceIntegrand, ) -> Callable[..., float]: + self._validate_integrand(integrand) + def scipy_integrand(*int_args: float) -> float: default = -1.0 # pylint: disable=R0801 @@ -42,6 +45,7 @@ def scipy_integrand(*int_args: float) -> float: return_val = integrand( mass, z, + self.sky_area, mass_proxy, z_proxy, self.mass_proxy_limits, @@ -55,6 +59,7 @@ def scipy_integrand(*int_args: float) -> float: def set_integration_bounds( self, cl_abundance: ClusterAbundance, + sky_area: float, z_proxy_limits: Tuple[float, float], mass_proxy_limits: Tuple[float, float], ) -> None: @@ -67,6 +72,7 @@ def set_integration_bounds( self.mass_proxy_limits = mass_proxy_limits self.z_proxy_limits = z_proxy_limits + self.sky_area = sky_area for kernel in cl_abundance.dirac_delta_kernels: if kernel.kernel_type == KernelType.Z_PROXY: @@ -90,7 +96,7 @@ def integrate( self, integrand: AbundanceIntegrand, ) -> float: - scipy_integrand = self._integral_wrapper(integrand) + scipy_integrand = self.integral_wrapper(integrand) val = nquad( scipy_integrand, ranges=self.integral_bounds, diff --git a/pytest.ini b/pytest.ini index c130f5b83..5cec28a1a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,7 +6,7 @@ addopts = testpaths = tests markers = slow: Mark slow tests to ignore them unless they are requested - + regression: Tests which cover the overall functionality of a feature. Typically carried out after significant changes. filterwarnings = ignore::DeprecationWarning:pkg_resources.*: ignore::DeprecationWarning:cobaya.*: diff --git a/tests/conftest.py b/tests/conftest.py index b052edd75..c1a8d204a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -165,7 +165,6 @@ def fixture_empty_cluster_abundance() -> ClusterAbundance: max_z=2, min_mass=13, max_mass=17, - sky_area=100, halo_mass_function=None, ) return cl_abundance diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 7390ee81e..0e7a5c2b6 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -85,7 +85,7 @@ def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): cosmo = pyccl.cosmology.CosmologyVanillaLCDM() params = ParamsMap() - tools.cluster_abundance = ClusterAbundance(13, 17, 0, 2, hmf, 4000) + tools.cluster_abundance = ClusterAbundance(13, 17, 0, 2, hmf) tools.update(params) tools.prepare(cosmo, params) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 740c40108..e9ca8ff4f 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -12,7 +12,7 @@ def fixture_cl_abundance(): """Test fixture that represents an assembled cluster abundance class.""" hmf = pyccl.halos.MassFuncBocquet16() - ca = ClusterAbundance(13, 17, 0, 2, hmf, 360.0**2) + ca = ClusterAbundance(13, 17, 0, 2, hmf) return ca @@ -24,8 +24,6 @@ def test_cluster_abundance_init(cl_abundance: ClusterAbundance): assert cl_abundance.max_mass == 17.0 assert cl_abundance.min_z == 0.0 assert cl_abundance.max_z == 2.0 - assert cl_abundance.sky_area == 360.0**2 - assert cl_abundance.sky_area_rad == 4 * np.pi**2 assert len(cl_abundance.kernels) == 0 @@ -59,15 +57,6 @@ def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): assert cl_abundance.cosmo == cosmo -def test_cluster_sky_area(cl_abundance: ClusterAbundance): - assert cl_abundance.sky_area == 360.0**2 - assert cl_abundance.sky_area_rad == 4 * np.pi**2 - - cl_abundance.sky_area = 180.0**2 - assert cl_abundance.sky_area == 180.0**2 - assert cl_abundance.sky_area_rad == np.pi**2 - - def test_cluster_add_kernel(cl_abundance: ClusterAbundance): assert len(cl_abundance.kernels) == 0 @@ -115,13 +104,14 @@ def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) - result = cl_abundance.comoving_volume(np.linspace(0.1, 1, 10)) + result = cl_abundance.comoving_volume(np.linspace(0.1, 1, 10), 360**2) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) assert len(result) == 10 assert np.all(result > 0) +@pytest.mark.regression def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -133,6 +123,7 @@ def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): assert np.all(result > 0) +@pytest.mark.regression def test_abundance_get_integrand(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -145,6 +136,7 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): mk.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(mk) + sky_area = 439.78986 mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) mass_proxy = np.linspace(0, 5, 5) @@ -154,11 +146,14 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): integrand = cl_abundance.get_integrand() assert callable(integrand) - result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) + result = integrand( + mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) +@pytest.mark.regression def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -171,6 +166,7 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): mk.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(mk) + sky_area = 439.78986 mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) mass_proxy = np.linspace(0, 5, 5) @@ -180,11 +176,14 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): integrand = cl_abundance.get_integrand(avg_mass=True) assert callable(integrand) - result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) + result = integrand( + mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) +@pytest.mark.regression def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -197,6 +196,7 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): mk.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(mk) + sky_area = 439.78986 mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) mass_proxy = np.linspace(0, 5, 5) @@ -206,11 +206,14 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): integrand = cl_abundance.get_integrand(avg_redshift=True) assert callable(integrand) - result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) + result = integrand( + mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) +@pytest.mark.regression def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cl_abundance.update_ingredients(cosmo, ParamsMap()) @@ -223,6 +226,7 @@ def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbun mk.distribution.return_value = np.atleast_1d(1.0) cl_abundance.add_kernel(mk) + sky_area = 439.78986 mass = np.linspace(13, 17, 5) z = np.linspace(0, 1, 5) mass_proxy = np.linspace(0, 5, 5) @@ -232,6 +236,8 @@ def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbun integrand = cl_abundance.get_integrand(avg_redshift=True, avg_mass=True) assert callable(integrand) - result = integrand(mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) + result = integrand( + mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) diff --git a/tests/test_cluster_integrators.py b/tests/test_cluster_integrators.py index 28e3fe0af..93f710ee7 100644 --- a/tests/test_cluster_integrators.py +++ b/tests/test_cluster_integrators.py @@ -8,7 +8,7 @@ from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator from firecrown.models.cluster.kernel import KernelType, Kernel -from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand @pytest.fixture(name="integrator", params=[ScipyIntegrator, NumCosmoIntegrator]) @@ -23,10 +23,11 @@ def test_numcosmo_set_integration_bounds_no_kernels( m_array = np.linspace(13, 17, 10) z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) + sky_area = 100**2 for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -46,6 +47,7 @@ def test_numcosmo_set_integration_bounds_dirac_delta( m_array = np.linspace(13, 17, 10) z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) + sky_area = 100**2 dd_kernel = Mock( spec=Kernel, @@ -58,7 +60,7 @@ def test_numcosmo_set_integration_bounds_dirac_delta( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -82,7 +84,7 @@ def test_numcosmo_set_integration_bounds_dirac_delta( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -104,7 +106,7 @@ def test_numcosmo_set_integration_bounds_dirac_delta( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -124,6 +126,7 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( m_array = np.linspace(13, 17, 10) z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) + sky_area = 100**2 ig_kernel = Mock( spec=Kernel, @@ -136,7 +139,7 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -161,7 +164,7 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -185,7 +188,7 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -207,6 +210,7 @@ def test_numcosmo_set_integration_bounds_analytic_slns( m_array = np.linspace(13, 17, 10) z_bins = list(zip(z_array[:-1], z_array[1:])) m_bins = list(zip(m_array[:-1], m_array[1:])) + sky_area = 100**2 a_kernel = Mock( spec=Kernel, @@ -219,7 +223,7 @@ def test_numcosmo_set_integration_bounds_analytic_slns( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -241,7 +245,7 @@ def test_numcosmo_set_integration_bounds_analytic_slns( for z_limits, mass_limits in zip(z_bins, m_bins): integrator.set_integration_bounds( - empty_cluster_abundance, z_limits, mass_limits + empty_cluster_abundance, sky_area, z_limits, mass_limits ) assert integrator.mass_proxy_limits == mass_limits @@ -261,21 +265,24 @@ def test_numcosmo_integrator_integrate( empty_cluster_abundance.max_mass = 1 empty_cluster_abundance.min_z = 0 empty_cluster_abundance.max_z = 1 + sky_area = 100**2 def integrand( a: npt.NDArray[np.float64], b: npt.NDArray[np.float64], - _c: npt.NDArray[np.float64], + _c: float, _d: npt.NDArray[np.float64], - _e: Tuple[float, float], + _e: npt.NDArray[np.float64], _f: Tuple[float, float], - ): + _g: Tuple[float, float], + ) -> npt.NDArray[np.float64]: # xy result = a * b return result integrator.set_integration_bounds( empty_cluster_abundance, + sky_area, (0, 1), (0, 1), ) From 7ebd12c6f478194d5b19de1460210ead4ce6b48a Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 28 Nov 2023 11:35:12 -0800 Subject: [PATCH 62/80] Unfortunately there's no easy way to get the mock to enforce number of arguments on methods in tests. --- .../statistic/binned_cluster_number_counts.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 5408b2594..d21f76301 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -119,7 +119,10 @@ def get_binned_cluster_masses( ): integrand = tools.cluster_abundance.get_integrand(avg_mass=True) self.integrator.set_integration_bounds( - tools.cluster_abundance, z_proxy_limits, mass_proxy_limits + tools.cluster_abundance, + self.sky_area, + z_proxy_limits, + mass_proxy_limits, ) total_mass = self.integrator.integrate(integrand) @@ -139,7 +142,10 @@ def get_binned_cluster_counts(self, tools: ModelingTools) -> List[float]: cluster_counts = [] for z_proxy_limits, mass_proxy_limits in self.bin_limits: self.integrator.set_integration_bounds( - tools.cluster_abundance, z_proxy_limits, mass_proxy_limits + tools.cluster_abundance, + self.sky_area, + z_proxy_limits, + mass_proxy_limits, ) integrand = tools.cluster_abundance.get_integrand() From 7070e12f36cadfd8feeb01a747fd5b94900bcf25 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 28 Nov 2023 12:04:00 -0800 Subject: [PATCH 63/80] Making cluster abundance updatable. Adjusting tests Adding some mypy type checking here and there --- firecrown/connector/cosmosis/likelihood.py | 6 +++--- .../gauss_family/statistic/supernova.py | 4 ++-- firecrown/likelihood/likelihood.py | 17 ++++++++++++++--- firecrown/modeling_tools.py | 9 +++------ firecrown/models/cluster/abundance.py | 11 ++++------- firecrown/parameters.py | 2 +- firecrown/updatable.py | 12 ++++++------ .../statistic/test_cluster_number_counts.py | 2 +- 8 files changed, 34 insertions(+), 29 deletions(-) diff --git a/firecrown/connector/cosmosis/likelihood.py b/firecrown/connector/cosmosis/likelihood.py index 2f2d6fdd3..c77a6cbee 100644 --- a/firecrown/connector/cosmosis/likelihood.py +++ b/firecrown/connector/cosmosis/likelihood.py @@ -117,7 +117,7 @@ def execute(self, sample: cosmosis.datablock) -> int: msg = self.form_error_message(exc) raise RuntimeError(msg) from exc - self.tools.prepare(ccl_cosmo, firecrown_params) + self.tools.prepare(ccl_cosmo) loglike = self.likelihood.compute_loglike(self.tools) derived_params_collection = self.likelihood.get_derived_parameters() @@ -173,7 +173,7 @@ def execute(self, sample: cosmosis.datablock) -> int: return 0 - def form_error_message(self, exc): + def form_error_message(self, exc: MissingSamplerParameterError) -> str: """Form the error message that will be used to report a missing parameter, when that parameter should have been supplied by the sampler.""" @@ -226,6 +226,6 @@ def execute(sample: cosmosis.datablock, instance: FirecrownLikelihood) -> int: return instance.execute(sample) -def cleanup(_): +def cleanup(_) -> int: """Cleanup hook for a CosmoSIS module. This one has nothing to do.""" return 0 diff --git a/firecrown/likelihood/gauss_family/statistic/supernova.py b/firecrown/likelihood/gauss_family/statistic/supernova.py index 986de4840..0c1a9e8aa 100644 --- a/firecrown/likelihood/gauss_family/statistic/supernova.py +++ b/firecrown/likelihood/gauss_family/statistic/supernova.py @@ -20,7 +20,7 @@ class Supernova(Statistic): """A statistic that applies an additive shift M to a supernova's distance modulus.""" - def __init__(self, sacc_tracer) -> None: + def __init__(self, sacc_tracer: str) -> None: """Initialize this statistic.""" super().__init__(parameter_prefix=sacc_tracer) @@ -29,7 +29,7 @@ def __init__(self, sacc_tracer) -> None: self.a: Optional[npt.NDArray[np.float64]] = None self.M = parameters.register_new_updatable_parameter() - def read(self, sacc_data: sacc.Sacc): + def read(self, sacc_data: sacc.Sacc) -> None: """Read the data for this statistic from the SACC file.""" # We do not actually need the tracer, but we want to make sure the SACC diff --git a/firecrown/likelihood/likelihood.py b/firecrown/likelihood/likelihood.py index 20331735a..d249be2b4 100644 --- a/firecrown/likelihood/likelihood.py +++ b/firecrown/likelihood/likelihood.py @@ -10,7 +10,7 @@ """ from __future__ import annotations -from typing import Mapping, Tuple, Union, Optional +from typing import Mapping, Tuple, Union, Optional, Set from abc import abstractmethod import types import warnings @@ -47,7 +47,7 @@ def __init__(self, parameter_prefix: Optional[str] = None) -> None: self.statistics: UpdatableCollection = UpdatableCollection() @abstractmethod - def read(self, sacc_data: sacc.Sacc): + def read(self, sacc_data: sacc.Sacc) -> None: """Read the covariance matrix for this likelihood from the SACC file.""" @abstractmethod @@ -142,7 +142,18 @@ def get_float_array(self, name: str) -> npt.NDArray[np.float64]: assert val.dtype == np.float64 return val - def to_set(self): + def to_set( + self, + ) -> Set[ + Union[ + str, + int, + bool, + float, + npt.NDArray[np.int64], + npt.NDArray[np.float64], + ] + ]: """Return the contained data as a set.""" return set(self.data) diff --git a/firecrown/modeling_tools.py b/firecrown/modeling_tools.py index ce13b9cad..caf58f411 100644 --- a/firecrown/modeling_tools.py +++ b/firecrown/modeling_tools.py @@ -9,7 +9,6 @@ from abc import ABC, abstractmethod import pyccl.nl_pt -from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance from .updatable import Updatable, UpdatableCollection @@ -35,7 +34,7 @@ def __init__( self._prepared: bool = False self.cluster_abundance = cluster_abundance - def add_pk(self, name: str, powerspectrum: pyccl.Pk2D): + def add_pk(self, name: str, powerspectrum: pyccl.Pk2D) -> None: """Add a :python:`pyccl.Pk2D` to the table of power spectra.""" if name in self.powerspectra: @@ -65,9 +64,7 @@ def has_pk(self, name: str) -> bool: return False return True - def prepare( - self, ccl_cosmo: pyccl.Cosmology, params: Optional[ParamsMap] = None - ) -> None: + def prepare(self, ccl_cosmo: pyccl.Cosmology) -> None: """Prepare the Cosmology for use in likelihoods. This method will prepare the ModelingTools for use in likelihoods. This @@ -95,7 +92,7 @@ def prepare( self.add_pk(name=pkm.name, powerspectrum=pkm.compute_p_of_k_z(tools=self)) if self.cluster_abundance is not None: - self.cluster_abundance.update_ingredients(ccl_cosmo, params) + self.cluster_abundance.update_ingredients(ccl_cosmo) self._prepared = True diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index bf8da19b5..393cad1cd 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -10,6 +10,7 @@ import pyccl import pyccl.background as bkg from pyccl.cosmology import Cosmology +from firecrown.updatable import Updatable, UpdatableCollection from firecrown.models.cluster.kernel import Kernel from firecrown.parameters import ParamsMap @@ -28,7 +29,7 @@ ] -class ClusterAbundance: +class ClusterAbundance(Updatable): """The class that calculates the predicted number counts of galaxy clusters The abundance is a function of a specific cosmology, a mass and redshift range, @@ -67,7 +68,8 @@ def __init__( max_z: float, halo_mass_function: pyccl.halos.MassFunc, ) -> None: - self.kernels: List[Kernel] = [] + super().__init__() + self.kernels: UpdatableCollection = UpdatableCollection() self.halo_mass_function = halo_mass_function self.min_mass = min_mass self.max_mass = max_mass @@ -86,11 +88,6 @@ def update_ingredients( """Update the cluster abundance calculation with a new cosmology.""" self._cosmo = cosmo self._hmf_cache = {} - if params is None: - return - - for kernel in self.kernels: - kernel.update(params) def comoving_volume( self, z: npt.NDArray[np.float64], sky_area: float = 0 diff --git a/firecrown/parameters.py b/firecrown/parameters.py index ced6f8e0f..25da404f7 100644 --- a/firecrown/parameters.py +++ b/firecrown/parameters.py @@ -91,7 +91,7 @@ def __len__(self): """Return the number of parameters contained.""" return len(self.params_names) - def __add__(self, other: RequiredParameters): + def __add__(self, other: RequiredParameters) -> RequiredParameters: """Return a new RequiredParameters with the concatenated names. Note that this function returns a new object that does not share state diff --git a/firecrown/updatable.py b/firecrown/updatable.py index b37667e77..235311384 100644 --- a/firecrown/updatable.py +++ b/firecrown/updatable.py @@ -14,7 +14,7 @@ """ from __future__ import annotations -from typing import final, Dict, Optional, Any, List, Union +from typing import final, Dict, Optional, Any, List, Union, Iterable from abc import ABC from collections import UserList from .parameters import ( @@ -300,7 +300,7 @@ def _get_derived_parameters(self) -> DerivedParameterCollection: return DerivedParameterCollection([]) -class UpdatableCollection(UserList): +class UpdatableCollection(UserList[Any]): """UpdatableCollection is a list of Updatable objects and is itself supports :python:`update` and :python:`reset` (although it does not inherit @@ -312,7 +312,7 @@ class UpdatableCollection(UserList): updated. """ - def __init__(self, iterable=None): + def __init__(self, iterable: Optional[Iterable[Any]] = None) -> None: """Initialize the UpdatableCollection from the supplied iterable. If the iterable contains any object that is not Updatable, a TypeError @@ -329,7 +329,7 @@ def __init__(self, iterable=None): ) @final - def update(self, params: ParamsMap): + def update(self, params: ParamsMap) -> None: """Update self by calling update() on each contained item. :param params: new parameter values @@ -350,7 +350,7 @@ def is_updated(self) -> bool: return self._updated @final - def reset(self): + def reset(self) -> None: """Resets self by calling reset() on each contained item.""" self._updated = False for updatable in self: @@ -394,7 +394,7 @@ def append(self, item: Updatable) -> None: ) super().append(item) - def __setitem__(self, key, value): + def __setitem__(self, key, value) -> None: """Set self[key] to value; raise TypeError if Value is not Updatable.""" if not isinstance(value, Updatable): raise TypeError( diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 0e7a5c2b6..3529a8159 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -87,7 +87,7 @@ def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): tools.cluster_abundance = ClusterAbundance(13, 17, 0, 2, hmf) tools.update(params) - tools.prepare(cosmo, params) + tools.prepare(cosmo) bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) bnc.read(cluster_sacc_data) From 6dc49dce91a46f24699b554956c684feeb75ceba Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 28 Nov 2023 13:31:11 -0800 Subject: [PATCH 64/80] Adding a new ClusterProperty flag --- .../cluster_redshift_richness.py | 16 +++---- .../statistic/binned_cluster_number_counts.py | 44 +++++++++++-------- firecrown/models/cluster/abundance.py | 23 +++++++--- firecrown/models/cluster/abundance_data.py | 7 ++- firecrown/models/cluster/properties.py | 9 ++++ .../statistic/test_cluster_number_counts.py | 42 +++++++++++------- tests/test_cluster_abundance.py | 40 +++++++++++++++-- tests/test_cluster_data.py | 21 +++++---- 8 files changed, 136 insertions(+), 66 deletions(-) create mode 100644 firecrown/models/cluster/properties.py diff --git a/examples/cluster_number_counts/cluster_redshift_richness.py b/examples/cluster_number_counts/cluster_redshift_richness.py index b46e6509d..a7acf0b3c 100644 --- a/examples/cluster_number_counts/cluster_redshift_richness.py +++ b/examples/cluster_number_counts/cluster_redshift_richness.py @@ -10,7 +10,7 @@ from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( BinnedClusterNumberCounts, ) -from firecrown.models.cluster.abundance_data import AbundanceData +from firecrown.models.cluster.properties import ClusterProperty from firecrown.modeling_tools import ModelingTools from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import ( @@ -48,15 +48,15 @@ def build_likelihood( # integrator = ScipyIntegrator() # Pull params for the likelihood from build_parameters - use_cluster_counts = build_parameters.get_bool("use_cluster_counts", True) - use_mean_log_mass = build_parameters.get_bool("use_mean_log_mass", False) + average_properties = ClusterProperty.NONE + if build_parameters.get_bool("use_cluster_counts", True): + average_properties |= ClusterProperty.COUNTS + if build_parameters.get_bool("use_mean_log_mass", False): + average_properties |= ClusterProperty.MASS + survey_name = "numcosmo_simulated_redshift_richness" likelihood = ConstGaussian( - [ - BinnedClusterNumberCounts( - use_cluster_counts, use_mean_log_mass, survey_name, integrator - ) - ] + [BinnedClusterNumberCounts(average_properties, survey_name, integrator)] ) # Read in sacc data diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index d21f76301..060391465 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -9,6 +9,7 @@ import numpy as np from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.models.cluster.abundance_data import AbundanceData +from firecrown.models.cluster.properties import ClusterProperty from firecrown.likelihood.gauss_family.statistic.statistic import ( Statistic, DataVector, @@ -27,8 +28,7 @@ class BinnedClusterNumberCounts(Statistic): def __init__( self, - cluster_counts: bool, - mean_log_mass: bool, + properties: ClusterProperty, survey_name: str, integrator: Integrator, systematics: Optional[List[SourceSystematic]] = None, @@ -36,8 +36,7 @@ def __init__( super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None - self.use_cluster_counts = cluster_counts - self.use_mean_log_mass = mean_log_mass + self.properties = properties self.survey_name = survey_name self.integrator = integrator self.data_vector = DataVector.from_list([]) @@ -51,17 +50,18 @@ def read(self, sacc_data: sacc.Sacc) -> None: sacc_indices = [] sacc_types = sacc.data_types.standard_types - sacc_adapter = AbundanceData( - sacc_data, self.survey_name, self.use_cluster_counts, self.use_mean_log_mass - ) + sacc_adapter = AbundanceData(sacc_data, self.survey_name, self.properties) - if self.use_cluster_counts: + if self.properties == ClusterProperty.NONE: + raise ValueError("You must specify at least one cluster property.") + + if ClusterProperty.COUNTS in self.properties: # pylint: disable=no-member data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) data_vector += data sacc_indices += indices - if self.use_mean_log_mass: + if ClusterProperty.MASS in self.properties: # pylint: disable=no-member data, indices = sacc_adapter.get_data_and_indices( sacc_types.cluster_mean_log_mass @@ -90,21 +90,27 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: theory_vector_list: List[float] = [] cluster_counts = [] - if not self.use_cluster_counts and not self.use_mean_log_mass: - return TheoryVector.from_list(theory_vector_list) - cluster_counts = self.get_binned_cluster_counts(tools) - if self.use_cluster_counts: - theory_vector_list += cluster_counts + for property in ClusterProperty: + if not property & self.properties: + continue - if self.use_mean_log_mass: - theory_vector_list += self.get_binned_cluster_masses(tools, cluster_counts) + if property == ClusterProperty.COUNTS: + theory_vector_list += cluster_counts + continue + + theory_vector_list += self.get_binned_cluster_property( + tools, cluster_counts, property + ) return TheoryVector.from_list(theory_vector_list) - def get_binned_cluster_masses( - self, tools: ModelingTools, cluster_counts: List[float] + def get_binned_cluster_property( + self, + tools: ModelingTools, + cluster_counts: List[float], + property: ClusterProperty, ) -> List[float]: """Computes the mean mass of clusters in each bin @@ -117,7 +123,7 @@ def get_binned_cluster_masses( for (z_proxy_limits, mass_proxy_limits), counts in zip( self.bin_limits, cluster_counts ): - integrand = tools.cluster_abundance.get_integrand(avg_mass=True) + integrand = tools.cluster_abundance.get_integrand(average=property) self.integrator.set_integration_bounds( tools.cluster_abundance, self.sky_area, diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 393cad1cd..a7ae570b0 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -13,6 +13,7 @@ from firecrown.updatable import Updatable, UpdatableCollection from firecrown.models.cluster.kernel import Kernel from firecrown.parameters import ParamsMap +from firecrown.models.cluster.properties import ClusterProperty AbundanceIntegrand = Callable[ @@ -130,7 +131,7 @@ def mass_function( return np.asarray(return_vals, dtype=np.float64) def get_integrand( - self, avg_mass: bool = False, avg_redshift: bool = False + self, *, average: Optional[ClusterProperty] = None ) -> AbundanceIntegrand: """Returns a callable that evaluates the complete integrand.""" @@ -145,15 +146,25 @@ def integrand( ) -> npt.NDArray[np.float64]: integrand = self.comoving_volume(z, sky_area) * self.mass_function(mass, z) - if avg_mass: - integrand *= mass - if avg_redshift: - integrand *= z - for kernel in self.kernels: + assert isinstance(kernel, Kernel) integrand *= kernel.distribution( mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits ) + + if average is None: + return integrand + + for prop in ClusterProperty: + if not prop & average: + continue + if prop == ClusterProperty.MASS: + integrand *= mass + elif prop == ClusterProperty.REDSHIFT: + integrand *= z + else: + raise NotImplementedError(f"Average for {prop}.") + return integrand return integrand diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 754a363f3..28f76df09 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -5,6 +5,7 @@ import numpy.typing as npt import sacc from sacc.tracers import SurveyTracer +from firecrown.models.cluster.properties import ClusterProperty class AbundanceData: @@ -24,12 +25,10 @@ def __init__( self, sacc_data: sacc.Sacc, survey_nm: str, - cluster_counts: bool, - mean_log_mass: bool, + properties: ClusterProperty, ): self.sacc_data = sacc_data - self.cluster_counts = cluster_counts - self.mean_log_mass = mean_log_mass + self.properties = properties try: self.survey_tracer: SurveyTracer = sacc_data.get_tracer(survey_nm) self.survey_nm = survey_nm diff --git a/firecrown/models/cluster/properties.py b/firecrown/models/cluster/properties.py new file mode 100644 index 000000000..9c2ac34e8 --- /dev/null +++ b/firecrown/models/cluster/properties.py @@ -0,0 +1,9 @@ +from enum import Flag, auto + + +class ClusterProperty(Flag): + NONE = 0 + COUNTS = auto() + MASS = auto() + REDSHIFT = auto() + SHEAR = auto() diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 3529a8159..ca6bd1af1 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -6,6 +6,7 @@ from firecrown.models.cluster.integrator.integrator import Integrator from firecrown.likelihood.gauss_family.statistic.source.source import SourceSystematic from firecrown.modeling_tools import ModelingTools +from firecrown.models.cluster.properties import ClusterProperty from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.likelihood.gauss_family.statistic.binned_cluster_number_counts import ( @@ -16,28 +17,31 @@ def test_create_binned_number_counts(): integrator = Mock(spec=Integrator) integrator.integrate.return_value = 1.0 - bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.NONE, "Test", integrator) assert bnc is not None - assert bnc.use_cluster_counts is False - assert bnc.use_mean_log_mass is False + assert bnc.properties == ClusterProperty.NONE assert bnc.survey_name == "Test" assert bnc.systematics == [] assert bnc.theory_vector is None assert len(bnc.data_vector) == 0 - bnc = BinnedClusterNumberCounts(True, True, "Test", integrator) - assert bnc.use_cluster_counts is True - assert bnc.use_mean_log_mass is True + bnc = BinnedClusterNumberCounts( + (ClusterProperty.COUNTS | ClusterProperty.MASS), "Test", integrator + ) + assert ClusterProperty.COUNTS in bnc.properties + assert ClusterProperty.MASS in bnc.properties systematics = [SourceSystematic("mock_systematic")] - bnc = BinnedClusterNumberCounts(False, False, "Test", integrator, systematics) + bnc = BinnedClusterNumberCounts( + ClusterProperty.NONE, "Test", integrator, systematics + ) assert bnc.systematics == systematics def test_get_data_vector(): integrator = Mock(spec=Integrator) integrator.integrate.return_value = 1.0 - bnc = BinnedClusterNumberCounts(False, False, "Test", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.NONE, "Test", integrator) dv = bnc.get_data_vector() assert dv is not None assert len(dv) == 0 @@ -46,29 +50,31 @@ def test_get_data_vector(): def test_read(cluster_sacc_data: sacc.Sacc): integrator = Mock(spec=Integrator) integrator.integrate.return_value = 1.0 - bnc = BinnedClusterNumberCounts(False, False, "my_survey", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.NONE, "my_survey", integrator) with pytest.raises( - RuntimeError, - match="has read a data vector of length 0; the length must be positive", + ValueError, + match="You must specify at least one cluster property", ): bnc.read(cluster_sacc_data) - bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.COUNTS, "my_survey", integrator) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 assert len(bnc.bin_limits) == 4 assert len(bnc.data_vector) == 2 assert len(bnc.sacc_indices) == 2 - bnc = BinnedClusterNumberCounts(False, True, "my_survey", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.MASS, "my_survey", integrator) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 assert len(bnc.bin_limits) == 4 assert len(bnc.data_vector) == 4 assert len(bnc.sacc_indices) == 4 - bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) + bnc = BinnedClusterNumberCounts( + (ClusterProperty.COUNTS | ClusterProperty.MASS), "my_survey", integrator + ) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 assert len(bnc.bin_limits) == 4 @@ -89,19 +95,21 @@ def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): tools.update(params) tools.prepare(cosmo) - bnc = BinnedClusterNumberCounts(True, False, "my_survey", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.COUNTS, "my_survey", integrator) bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None assert len(tv) == 4 - bnc = BinnedClusterNumberCounts(False, True, "my_survey", integrator) + bnc = BinnedClusterNumberCounts(ClusterProperty.MASS, "my_survey", integrator) bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None assert len(tv) == 4 - bnc = BinnedClusterNumberCounts(True, True, "my_survey", integrator) + bnc = BinnedClusterNumberCounts( + (ClusterProperty.COUNTS | ClusterProperty.MASS), "my_survey", integrator + ) bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index e9ca8ff4f..0c21c4a97 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -6,6 +6,7 @@ from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import Kernel, KernelType +from firecrown.models.cluster.properties import ClusterProperty @pytest.fixture(name="cl_abundance") @@ -174,7 +175,7 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): mass_proxy_limits = (0, 5) z_proxy_limits = (0, 1) - integrand = cl_abundance.get_integrand(avg_mass=True) + integrand = cl_abundance.get_integrand(average=ClusterProperty.MASS) assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits @@ -204,7 +205,7 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): mass_proxy_limits = (0, 5) z_proxy_limits = (0, 1) - integrand = cl_abundance.get_integrand(avg_redshift=True) + integrand = cl_abundance.get_integrand(average=ClusterProperty.REDSHIFT) assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits @@ -234,10 +235,43 @@ def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbun mass_proxy_limits = (0, 5) z_proxy_limits = (0, 1) - integrand = cl_abundance.get_integrand(avg_redshift=True, avg_mass=True) + average_properties = ClusterProperty.MASS | ClusterProperty.REDSHIFT + integrand = cl_abundance.get_integrand(average=average_properties) assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits ) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) + + +@pytest.mark.regression +def test_abundance_get_integrand_avg_not_implemented_throws( + cl_abundance: ClusterAbundance, +): + cosmo = pyccl.CosmologyVanillaLCDM() + cl_abundance.update_ingredients(cosmo, ParamsMap()) + mk = Mock( + spec=Kernel, + kernel_type=KernelType.MASS, + is_dirac_delta=False, + has_analytic_sln=False, + ) + mk.distribution.return_value = np.atleast_1d(1.0) + cl_abundance.add_kernel(mk) + + sky_area = 439.78986 + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + + average_properties = ClusterProperty.SHEAR + integrand = cl_abundance.get_integrand(average=average_properties) + assert callable(integrand) + with pytest.raises(NotImplementedError): + _ = integrand( + mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits + ) diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index c5a729c02..5049b13de 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -3,29 +3,32 @@ import numpy as np import sacc from firecrown.models.cluster.abundance_data import AbundanceData +from firecrown.models.cluster.properties import ClusterProperty def test_create_abundance_data_no_survey(): with pytest.raises( ValueError, match="The SACC file does not contain the SurveyTracer" ): - _ = AbundanceData(sacc.Sacc(), "survey", False, False) + _ = AbundanceData(sacc.Sacc(), "survey", ClusterProperty.NONE) def test_create_abundance_data_wrong_tracer(): s = sacc.Sacc() s.add_tracer("bin_richness", "test", 0.1, 0.2) with pytest.raises(ValueError, match="The SACC tracer test is not a SurveyTracer"): - _ = AbundanceData(s, "test", False, False) + _ = AbundanceData(s, "test", ClusterProperty.NONE) def test_create_abundance_data(): s = sacc.Sacc() s.add_tracer("survey", "mock_survey", 4000) - ad = AbundanceData(s, "mock_survey", True, True) + ad = AbundanceData( + s, "mock_survey", (ClusterProperty.COUNTS | ClusterProperty.MASS) + ) - assert ad.cluster_counts is True - assert ad.mean_log_mass is True + assert ClusterProperty.COUNTS in ad.properties + assert ClusterProperty.MASS in ad.properties assert ad.survey_nm == "mock_survey" assert ad.survey_tracer.sky_area == 4000 # pylint: disable=protected-access @@ -44,7 +47,7 @@ def test_validate_tracers(): s.add_data_point( sacc.standard_types.cluster_counts, ("mock_survey", "my_tracer"), 1 ) - ad = AbundanceData(s, "mock_survey", False, False) + ad = AbundanceData(s, "mock_survey", ClusterProperty.NONE) # pylint: disable=no-member tracer_combs = np.array( s.get_tracer_combinations(sacc.standard_types.cluster_mean_log_mass) @@ -68,7 +71,7 @@ def test_validate_tracers(): def test_filtered_tracers(cluster_sacc_data): - ad = AbundanceData(cluster_sacc_data, "my_survey", False, False) + ad = AbundanceData(cluster_sacc_data, "my_survey", ClusterProperty.NONE) # pylint: disable=no-member cc = sacc.standard_types.cluster_counts filtered_tracers, survey_mask = ad.get_filtered_tracers(cc) @@ -81,7 +84,7 @@ def test_filtered_tracers(cluster_sacc_data): def test_get_data_and_indices(cluster_sacc_data): - ad = AbundanceData(cluster_sacc_data, "my_survey", False, False) + ad = AbundanceData(cluster_sacc_data, "my_survey", ClusterProperty.NONE) # pylint: disable=no-member cc = sacc.standard_types.cluster_counts data, indices = ad.get_data_and_indices(cc) @@ -91,7 +94,7 @@ def test_get_data_and_indices(cluster_sacc_data): def test_get_bin_limits(cluster_sacc_data): - ad = AbundanceData(cluster_sacc_data, "my_survey", False, False) + ad = AbundanceData(cluster_sacc_data, "my_survey", ClusterProperty.NONE) # pylint: disable=no-member cc = sacc.standard_types.cluster_counts limits = ad.get_bin_limits(cc) From 6676992433a96828d9c9e50a7f59f28048001c62 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 29 Nov 2023 12:23:31 -0800 Subject: [PATCH 65/80] Updating the comparison script for numcosmo integration methods. Allowing a user to pass in an integration method into the num cosmo integrator. --- .../compare_integration_methods.py | 130 ++++++++++-------- .../cluster/integrator/numcosmo_integrator.py | 18 ++- 2 files changed, 87 insertions(+), 61 deletions(-) diff --git a/examples/cluster_number_counts/compare_integration_methods.py b/examples/cluster_number_counts/compare_integration_methods.py index f6b135dbb..fb5563408 100644 --- a/examples/cluster_number_counts/compare_integration_methods.py +++ b/examples/cluster_number_counts/compare_integration_methods.py @@ -1,26 +1,20 @@ """Test integral methods for cluster abundance.""" - -from typing import Any, Dict import time import itertools -import pyccl as ccl +import pyccl import numpy as np -from numcosmo_py import Ncm - -from firecrown.models.cluster_abundance_old import ClusterAbundance -from firecrown.models.cluster_mass_rich_proxy import ( - ClusterMassRich, - ClusterMassRichBinArgument, -) -from firecrown.models.cluster_redshift_spec import ( - ClusterRedshiftSpecArgument, +from firecrown.models.cluster.mass_proxy import MurataBinned +from firecrown.models.cluster.kernel import Kernel +from firecrown.models.cluster.integrator.numcosmo_integrator import ( + NumCosmoIntegrator, + NumCosmoIntegralMethod, ) +from firecrown.models.cluster.abundance import ClusterAbundance +from firecrown.models.cluster.kernel import SpectroscopicRedshift -def compare_integration(): - """Compare integration methods.""" - +def get_cosmology() -> pyccl.Cosmology: Omega_c = 0.262 Omega_b = 0.049 Omega_k = 0.0 @@ -32,7 +26,8 @@ def compare_integration(): Neff = 3.046 w0 = -1.0 wa = 0.0 - cosmo_ccl = ccl.Cosmology( + + cosmo_ccl = pyccl.Cosmology( Omega_c=Omega_c, Omega_b=Omega_b, Neff=Neff, @@ -45,56 +40,75 @@ def compare_integration(): T_CMB=Tcmb0, m_nu=[0.00, 0.0, 0.0], ) + + return cosmo_ccl + + +def get_mass_richness() -> Kernel: pivot_mass = 14.625862906 pivot_redshift = 0.6 - cluster_mass_r = ClusterMassRich(pivot_mass, pivot_redshift) - - cluster_mass_r.mu_p0 = 3.0 - cluster_mass_r.mu_p1 = 0.86 - cluster_mass_r.mu_p2 = 0.0 - cluster_mass_r.sigma_p0 = 3.0 - cluster_mass_r.sigma_p1 = 0.7 - cluster_mass_r.sigma_p2 = 0.0 - - # TODO: remove try/except when pyccl 3.0 is released - try: - hmd_200 = ccl.halos.MassDef200m() - except TypeError: - hmd_200 = ccl.halos.MassDef200m - - hmf_args: Dict[str, Any] = {} - hmf_name = "Tinker08" - z_bins = np.linspace(0.0, 1.0, 4) - r_bins = np.linspace(1.0, 2.5, 5) - - integ_options = [ - Ncm.IntegralNDMethod.P, - Ncm.IntegralNDMethod.P_V, - Ncm.IntegralNDMethod.H, - Ncm.IntegralNDMethod.H_V, - ] - for integ_method in integ_options: - abundance_test = ClusterAbundance( - hmd_200, hmf_name, hmf_args, integ_method=integ_method - ) - test_m_list = [] - t1 = time.time() + mass_richness = MurataBinned(pivot_mass, pivot_redshift) + + mass_richness.mu_p0 = 3.0 + mass_richness.mu_p1 = 0.86 + mass_richness.mu_p2 = 0.0 + mass_richness.sigma_p0 = 3.0 + mass_richness.sigma_p1 = 0.7 + mass_richness.sigma_p2 = 0.0 + + return mass_richness + - for i, j in itertools.product(range(3), range(4)): - cluster_mass_bin = ClusterMassRichBinArgument( - cluster_mass_r, 13, 15, r_bins[j], r_bins[j + 1] +def compare_integration() -> None: + """Compare integration methods.""" + hmf = pyccl.halos.MassFuncTinker08() + abundance = ClusterAbundance(13, 15, 0, 4, hmf) + + mass_richness = get_mass_richness() + abundance.add_kernel(mass_richness) + + redshift_proxy_kernel = SpectroscopicRedshift() + abundance.add_kernel(redshift_proxy_kernel) + + cosmo = get_cosmology() + abundance.update_ingredients(cosmo) + + sky_area = 360**2 + integrand = abundance.get_integrand() + + for method in NumCosmoIntegralMethod: + counts_list = [] + t_start = time.time() + + nc_integrator = NumCosmoIntegrator(method=method) + nc_integrator.set_integration_bounds(abundance, 496, (0, 4), (13, 15)) + + z_bins = np.linspace(0.0, 1.0, 4) + mass_proxy_bins = np.linspace(1.0, 2.5, 5) + + for z_idx, mass_proxy_idx in itertools.product(range(3), range(4)): + z_proxy_limits = (z_bins[z_idx], z_bins[z_idx + 1]) + mass_proxy_limits = ( + mass_proxy_bins[mass_proxy_idx], + mass_proxy_bins[mass_proxy_idx + 1], ) - cluster_z_bin = ClusterRedshiftSpecArgument(z_bins[i], z_bins[i + 1]) - cluster_counts = abundance_test.compute( - cosmo_ccl, cluster_mass_bin, cluster_z_bin + + nc_integrator.set_integration_bounds( + abundance, + sky_area, + z_proxy_limits, + mass_proxy_limits, ) - test_m_list.append(cluster_counts) - t2 = time.time() + + counts = nc_integrator.integrate(integrand) + counts_list.append(counts) + + t_stop = time.time() print( - f"The time for {integ_method} is {t2-t1}\n\n" - f"The counts value is {test_m_list}\n\n" + f"The time for NumCosmo integration method {method} is {t_stop-t_start}\n\n" + f"The counts value is {counts_list}\n\n" ) diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 7b3335493..b078753a5 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -2,7 +2,8 @@ This module holds the NumCosmo implementation of the integrator classes """ -from typing import Tuple, Callable, Dict, Sequence, List +from typing import Tuple, Callable, Dict, Sequence, List, Optional +from enum import Enum import numpy as np import numpy.typing as npt from numcosmo_py import Ncm @@ -11,15 +12,26 @@ from firecrown.models.cluster.integrator.integrator import Integrator +class NumCosmoIntegralMethod(Enum): + P = Ncm.IntegralNDMethod.P + P_V = Ncm.IntegralNDMethod.P_V + H = Ncm.IntegralNDMethod.H + H_V = Ncm.IntegralNDMethod.H_V + + class NumCosmoIntegrator(Integrator): """The NumCosmo implementation of the Integrator base class.""" def __init__( - self, relative_tolerance: float = 1e-4, absolute_tolerance: float = 1e-12 + self, + method: Optional[NumCosmoIntegralMethod] = None, + relative_tolerance: float = 1e-4, + absolute_tolerance: float = 1e-12, ) -> None: super().__init__() self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance + self._method = method or NumCosmoIntegralMethod.P_V self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() self.integral_bounds: List[Tuple[float, float]] = [] @@ -100,7 +112,7 @@ def integrate( Ncm.cfg_init() ncm_integrand = self._integral_wrapper(integrand) int_nd = CountsIntegralND(len(self.integral_bounds), ncm_integrand) - int_nd.set_method(Ncm.IntegralNDMethod.P_V) + int_nd.set_method(self._method.value) int_nd.set_reltol(self._relative_tolerance) int_nd.set_abstol(self._absolute_tolerance) res = Ncm.Vector.new(1) From 69b337b9b268acbfec340fa09a2174c743f2c361 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 1 Dec 2023 13:20:20 -0800 Subject: [PATCH 66/80] Adding new NDimensionalBin class and SaccBin implementation. Updating the cluster abundance data to return SaccBins instead of confusing built in types Adding tests, and updating sacc file to reflect proper structure Fixing mypy/pylint complaints. --- .../statistic/binned_cluster_number_counts.py | 78 ++++---- firecrown/models/cluster/abundance.py | 5 +- firecrown/models/cluster/abundance_data.py | 132 +++++++++----- firecrown/models/cluster/binning.py | 71 ++++++++ .../models/cluster/integrator/integrator.py | 2 +- .../cluster/integrator/numcosmo_integrator.py | 2 + .../cluster/integrator/scipy_integrator.py | 4 +- firecrown/models/cluster/properties.py | 4 + tests/conftest.py | 27 +-- .../statistic/test_cluster_number_counts.py | 26 +-- tests/test_cluster_data.py | 167 ++++++++++-------- tests/test_cluster_integrators.py | 2 +- 12 files changed, 319 insertions(+), 201 deletions(-) create mode 100644 firecrown/models/cluster/binning.py diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 060391465..676228614 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -4,11 +4,11 @@ clusters within a single redshift and mass bin. """ from __future__ import annotations -from typing import List, Optional, Tuple +from typing import List, Optional import sacc import numpy as np from firecrown.models.cluster.integrator.integrator import Integrator -from firecrown.models.cluster.abundance_data import AbundanceData +from firecrown.models.cluster.abundance_data import AbundanceData, SaccBin from firecrown.models.cluster.properties import ClusterProperty from firecrown.likelihood.gauss_family.statistic.statistic import ( Statistic, @@ -28,7 +28,7 @@ class BinnedClusterNumberCounts(Statistic): def __init__( self, - properties: ClusterProperty, + cluster_properties: ClusterProperty, survey_name: str, integrator: Integrator, systematics: Optional[List[SourceSystematic]] = None, @@ -36,48 +36,32 @@ def __init__( super().__init__() self.systematics = systematics or [] self.theory_vector: Optional[TheoryVector] = None - self.properties = properties + self.cluster_properties = cluster_properties self.survey_name = survey_name self.integrator = integrator self.data_vector = DataVector.from_list([]) self.sky_area = 0.0 - self.bin_limits: List[List[Tuple[float, float]]] = [] + self.bin_edges: List[SaccBin] = [] + self.bin_dimensions = 2 def read(self, sacc_data: sacc.Sacc) -> None: # Build the data vector and indices needed for the likelihood - - data_vector = [] - sacc_indices = [] - - sacc_types = sacc.data_types.standard_types - sacc_adapter = AbundanceData(sacc_data, self.survey_name, self.properties) - - if self.properties == ClusterProperty.NONE: + if self.cluster_properties == ClusterProperty.NONE: raise ValueError("You must specify at least one cluster property.") - if ClusterProperty.COUNTS in self.properties: - # pylint: disable=no-member - data, indices = sacc_adapter.get_data_and_indices(sacc_types.cluster_counts) - data_vector += data - sacc_indices += indices + sacc_adapter = AbundanceData(sacc_data, self.bin_dimensions) + self.sky_area = sacc_adapter.get_survey_tracer(self.survey_name).sky_area - if ClusterProperty.MASS in self.properties: - # pylint: disable=no-member - data, indices = sacc_adapter.get_data_and_indices( - sacc_types.cluster_mean_log_mass - ) - data_vector += data - sacc_indices += indices - - self.sky_area = sacc_adapter.survey_tracer.sky_area - # Note - this is the same for both cl mass and cl counts... Why do we need to - # specify a data type? + data, indices = sacc_adapter.get_observed_data_and_indices_by_survey( + self.survey_name, self.cluster_properties + ) + self.data_vector = DataVector.from_list(data) + self.sacc_indices = np.array(indices) - # pylint: disable=no-member - self.bin_limits = sacc_adapter.get_bin_limits(sacc_types.cluster_mean_log_mass) - self.data_vector = DataVector.from_list(data_vector) + self.bin_edges = sacc_adapter.get_bin_edges( + self.survey_name, self.cluster_properties + ) - self.sacc_indices = np.array(sacc_indices) super().read(sacc_data) def get_data_vector(self) -> DataVector: @@ -92,16 +76,16 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_counts = self.get_binned_cluster_counts(tools) - for property in ClusterProperty: - if not property & self.properties: + for cl_property in ClusterProperty: + if not cl_property & self.cluster_properties: continue - if property == ClusterProperty.COUNTS: + if cl_property == ClusterProperty.COUNTS: theory_vector_list += cluster_counts continue theory_vector_list += self.get_binned_cluster_property( - tools, cluster_counts, property + tools, cluster_counts, cl_property ) return TheoryVector.from_list(theory_vector_list) @@ -110,7 +94,7 @@ def get_binned_cluster_property( self, tools: ModelingTools, cluster_counts: List[float], - property: ClusterProperty, + cluster_properties: ClusterProperty, ) -> List[float]: """Computes the mean mass of clusters in each bin @@ -120,15 +104,15 @@ def get_binned_cluster_property( assert tools.cluster_abundance is not None cluster_masses = [] - for (z_proxy_limits, mass_proxy_limits), counts in zip( - self.bin_limits, cluster_counts - ): - integrand = tools.cluster_abundance.get_integrand(average=property) + for bin_edge, counts in zip(self.bin_edges, cluster_counts): + integrand = tools.cluster_abundance.get_integrand( + average=cluster_properties + ) self.integrator.set_integration_bounds( tools.cluster_abundance, self.sky_area, - z_proxy_limits, - mass_proxy_limits, + bin_edge.z_edges, + bin_edge.mass_proxy_edges, ) total_mass = self.integrator.integrate(integrand) @@ -146,12 +130,12 @@ def get_binned_cluster_counts(self, tools: ModelingTools) -> List[float]: assert tools.cluster_abundance is not None cluster_counts = [] - for z_proxy_limits, mass_proxy_limits in self.bin_limits: + for bin_edge in self.bin_edges: self.integrator.set_integration_bounds( tools.cluster_abundance, self.sky_area, - z_proxy_limits, - mass_proxy_limits, + bin_edge.z_edges, + bin_edge.mass_proxy_edges, ) integrand = tools.cluster_abundance.get_integrand() diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index a7ae570b0..d63d0ba36 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -12,7 +12,6 @@ from pyccl.cosmology import Cosmology from firecrown.updatable import Updatable, UpdatableCollection from firecrown.models.cluster.kernel import Kernel -from firecrown.parameters import ParamsMap from firecrown.models.cluster.properties import ClusterProperty @@ -83,9 +82,7 @@ def add_kernel(self, kernel: Kernel) -> None: """Add a kernel to the cluster abundance integrand""" self.kernels.append(kernel) - def update_ingredients( - self, cosmo: Cosmology, params: Optional[ParamsMap] = None - ) -> None: + def update_ingredients(self, cosmo: Cosmology) -> None: """Update the cluster abundance calculation with a new cosmology.""" self._cosmo = cosmo self._hmf_cache = {} diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 28f76df09..2fc62e48e 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -6,12 +6,13 @@ import sacc from sacc.tracers import SurveyTracer from firecrown.models.cluster.properties import ClusterProperty +from firecrown.models.cluster.binning import SaccBin class AbundanceData: """The class used to wrap a sacc file and return the cluster abundance data. - The sacc file is a complicated set of tracers and surveys. This class + The sacc file is a complicated set of tracers (bins) and surveys. This class manipulates that data and returns only the data relevant for the cluster number count statistic. The data in this class is specific to a single survey name.""" @@ -24,69 +25,116 @@ class AbundanceData: def __init__( self, sacc_data: sacc.Sacc, - survey_nm: str, - properties: ClusterProperty, + bin_dimensions: int, ): self.sacc_data = sacc_data - self.properties = properties + self.bin_dimensions = bin_dimensions + + def get_survey_tracer(self, survey_nm: str) -> SurveyTracer: + """Returns the SurveyTracer for the specified survey name.""" try: - self.survey_tracer: SurveyTracer = sacc_data.get_tracer(survey_nm) - self.survey_nm = survey_nm + survey_tracer: SurveyTracer = self.sacc_data.get_tracer(survey_nm) except KeyError as exc: raise ValueError( f"The SACC file does not contain the SurveyTracer " f"{survey_nm}." ) from exc - if not isinstance(self.survey_tracer, SurveyTracer): + + if not isinstance(survey_tracer, SurveyTracer): raise ValueError(f"The SACC tracer {survey_nm} is not a SurveyTracer.") - def get_filtered_tracers(self, data_type: str) -> Tuple[npt.NDArray, npt.NDArray]: - """Returns only tracers that match the data type requested.""" - all_tracers = np.array( + return survey_tracer + + def get_observed_data_and_indices_by_survey( + self, + survey_nm: str, + properties: ClusterProperty, + ) -> Tuple[List[float], List[int]]: + """Will return observed data (and sacc indices) for a specified survey based on + the properties requested by the caller. + + For example if the caller has enabled COUNTS then the observed cluster counts + within each N dimensional bin will be returned.""" + + data_vectors = [] + sacc_indices = [] + + for cluster_property in ClusterProperty: + if cluster_property not in properties: + continue + + if cluster_property == ClusterProperty.COUNTS: + # pylint: disable=no-member + data_type = sacc.standard_types.cluster_counts + elif cluster_property == ClusterProperty.MASS: + # pylint: disable=no-member + data_type = sacc.standard_types.cluster_mean_log_mass + else: + raise NotImplementedError(cluster_property) + + bin_combinations = self._all_bin_combinations_for_data_type(data_type) + + my_survey_mask = bin_combinations[:, self._survey_index] == survey_nm + + data_vectors += list( + self.sacc_data.get_mean(data_type=data_type)[my_survey_mask] + ) + + sacc_indices += list( + self.sacc_data.indices(data_type=data_type)[my_survey_mask] + ) + + return data_vectors, sacc_indices + + def _all_bin_combinations_for_data_type(self, data_type: str) -> npt.NDArray: + bins_combos_for_type = np.array( self.sacc_data.get_tracer_combinations(data_type=data_type) ) - self.validate_tracers(all_tracers, data_type) - survey_mask = all_tracers[:, self._survey_index] == self.survey_nm - filtered_tracers = all_tracers[survey_mask] - return filtered_tracers, survey_mask - - def get_data_and_indices(self, data_type: str) -> Tuple[List[float], List[int]]: - """Returns the data vector and indices for the requested data type.""" - _, survey_mask = self.get_filtered_tracers(data_type) - data_vector_list = list( - self.sacc_data.get_mean(data_type=data_type)[survey_mask] - ) - sacc_indices_list = list( - self.sacc_data.indices(data_type=data_type)[survey_mask] - ) - return data_vector_list, sacc_indices_list - def validate_tracers( - self, tracers_combinations: npt.NDArray, data_type: str - ) -> None: - """Validates that the tracers requested exist and are valid.""" - if len(tracers_combinations) == 0: + if len(bins_combos_for_type) == 0: raise ValueError( f"The SACC file does not contain any tracers for the " f"{data_type} data type." ) - if tracers_combinations.shape[1] != 3: + # Add one because we don't include survey in the bin dimensions (for now) + if bins_combos_for_type.shape[1] != self.bin_dimensions + 1: raise ValueError( "The SACC file must contain 3 tracers for the " "cluster_counts data type: cluster_survey, " "redshift argument and mass argument tracers." ) - def get_bin_limits(self, data_type: str) -> List[List[Tuple[float, float]]]: + return bins_combos_for_type + + def get_bin_edges( + self, survey_nm: str, properties: ClusterProperty + ) -> List[SaccBin]: """Returns the limits for all z, mass bins for the requested data type.""" - filtered_tracers, _ = self.get_filtered_tracers(data_type) - - tracers = [] - for _, z_tracer, mass_tracer in filtered_tracers: - z_data: sacc.BaseTracer = self.sacc_data.get_tracer(z_tracer) - mass_data: sacc.BaseTracer = self.sacc_data.get_tracer(mass_tracer) - tracers.append( - [(z_data.lower, z_data.upper), (mass_data.lower, mass_data.upper)] - ) + bins = [] + + for cluster_property in ClusterProperty: + if cluster_property not in properties: + continue + + if cluster_property == ClusterProperty.COUNTS: + # pylint: disable=no-member + data_type = sacc.standard_types.cluster_counts + elif cluster_property == ClusterProperty.MASS: + # pylint: disable=no-member + data_type = sacc.standard_types.cluster_mean_log_mass + else: + raise NotImplementedError(cluster_property) + + bin_combinations = self._all_bin_combinations_for_data_type(data_type) + my_survey_mask = bin_combinations[:, self._survey_index] == survey_nm + bin_combinations_for_survey = bin_combinations[my_survey_mask] + + for _, z_tracer, mass_tracer in bin_combinations_for_survey: + z_data: sacc.BaseTracer = self.sacc_data.get_tracer(z_tracer) + mass_data: sacc.BaseTracer = self.sacc_data.get_tracer(mass_tracer) + sacc_bin = SaccBin([z_data, mass_data], self.bin_dimensions) + bins.append(sacc_bin) - return tracers + # Remove duplicates while preserving order (i.e. dont use set()) + unique_bins = list(dict.fromkeys(bins)) + return unique_bins diff --git a/firecrown/models/cluster/binning.py b/firecrown/models/cluster/binning.py new file mode 100644 index 000000000..a5b34343d --- /dev/null +++ b/firecrown/models/cluster/binning.py @@ -0,0 +1,71 @@ +"""This module contains the classes that define the bins and binning +used for cluster theoretical predictions within Firecrown.""" + +from typing import Tuple, List, TypeVar, Generic +from abc import ABC, abstractmethod +import sacc + +T = TypeVar("T") + + +class NDimensionalBin(Generic[T], ABC): + """Class which defines the interface for an N dimensional bin used in + the cluster likelihood.""" + + def __init__(self, bins: List[T], dimension: int): + self.bins = bins + self.dimension = dimension + assert len(bins) == dimension + + @property + @abstractmethod + def z_edges(self) -> Tuple[float, float]: + """Redshift bin edges""" + + @property + @abstractmethod + def mass_proxy_edges(self) -> Tuple[float, float]: + """Mass proxy bin edges""" + + def __str__(self) -> str: + return f"[{self.z_edges}, {self.mass_proxy_edges}]\n" + + def __repr__(self) -> str: + return f"[{self.z_edges}, {self.mass_proxy_edges}]\n" + + +class SaccBin(NDimensionalBin[sacc.BaseTracer]): + """An implementation of the N dimensional bin using sacc tracers.""" + + def __init__(self, bins: List[sacc.BaseTracer], dimension: int): + super().__init__(bins, dimension) + + @property + def z_edges(self) -> Tuple[float, float]: + return self.bins[0].lower, self.bins[0].upper + + @property + def mass_proxy_edges(self) -> Tuple[float, float]: + return self.bins[1].lower, self.bins[1].upper + + def __eq__(self, other: object) -> bool: + """Two bins are equal if they have the same lower/upper bound.""" + if not isinstance(other, SaccBin): + return False + + if self.dimension != other.dimension: + return False + + for i, my_bin in enumerate(self.bins): + other_bin = other.bins[i] + if my_bin.lower != other_bin.lower: + return False + if my_bin.upper != other_bin.upper: + return False + + return True + + def __hash__(self) -> int: + """One bin's hash is determined by the dimension and lower/upper bound.""" + bin_bounds = [(bin.lower, bin.upper) for bin in self.bins] + return hash((self.dimension, tuple(bin_bounds))) diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index 40292c40b..a28745156 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -5,7 +5,7 @@ """ import inspect from abc import ABC, abstractmethod -from typing import Tuple, Dict, Callable, get_args +from typing import Tuple, Dict, get_args from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand from firecrown.models.cluster.kernel import KernelType diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index b078753a5..3a07b7a08 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -13,6 +13,8 @@ class NumCosmoIntegralMethod(Enum): + """The available NumCosmo integration methods.""" + P = Ncm.IntegralNDMethod.P P_V = Ncm.IntegralNDMethod.P_V H = Ncm.IntegralNDMethod.H diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index 2120b762a..003858491 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -28,7 +28,7 @@ def __init__( self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) self.sky_area: float = 360**2 - def integral_wrapper( + def _integral_wrapper( self, integrand: AbundanceIntegrand, ) -> Callable[..., float]: @@ -96,7 +96,7 @@ def integrate( self, integrand: AbundanceIntegrand, ) -> float: - scipy_integrand = self.integral_wrapper(integrand) + scipy_integrand = self._integral_wrapper(integrand) val = nquad( scipy_integrand, ranges=self.integral_bounds, diff --git a/firecrown/models/cluster/properties.py b/firecrown/models/cluster/properties.py index 9c2ac34e8..e141b13b9 100644 --- a/firecrown/models/cluster/properties.py +++ b/firecrown/models/cluster/properties.py @@ -1,7 +1,11 @@ +"""Module containing classes relevant to defining cluster properties.""" from enum import Flag, auto class ClusterProperty(Flag): + """Flag containing the possible cluster properties we can make a theoretical + prediction for.""" + NONE = 0 COUNTS = auto() MASS = auto() diff --git a/tests/conftest.py b/tests/conftest.py index c1a8d204a..966d018a5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -141,19 +141,20 @@ def fixture_cluster_sacc_data() -> sacc.Sacc: s = sacc.Sacc() s.add_tracer("survey", "my_survey", 4000) s.add_tracer("survey", "not_my_survey", 4000) - s.add_tracer("bin_z", "my_tracer1", 0, 2) - s.add_tracer("bin_z", "my_tracer2", 2, 4) - s.add_tracer("bin_richness", "my_other_tracer1", 0, 2) - s.add_tracer("bin_richness", "my_other_tracer2", 2, 4) - - s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) - s.add_data_point(cc, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) - s.add_data_point(cc, ("not_my_survey", "my_tracer1", "my_other_tracer2"), 1) - - s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer1"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer1", "my_other_tracer2"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer2"), 1) - s.add_data_point(mlm, ("my_survey", "my_tracer2", "my_other_tracer1"), 1) + s.add_tracer("bin_z", "z_bin_tracer_1", 0, 2) + s.add_tracer("bin_z", "z_bin_tracer_2", 2, 4) + s.add_tracer("bin_richness", "mass_bin_tracer_1", 0, 2) + s.add_tracer("bin_richness", "mass_bin_tracer_2", 2, 4) + + s.add_data_point(cc, ("my_survey", "z_bin_tracer_1", "mass_bin_tracer_1"), 1) + s.add_data_point(cc, ("my_survey", "z_bin_tracer_1", "mass_bin_tracer_2"), 1) + s.add_data_point(cc, ("not_my_survey", "z_bin_tracer_2", "mass_bin_tracer_1"), 1) + s.add_data_point(cc, ("not_my_survey", "z_bin_tracer_2", "mass_bin_tracer_2"), 1) + + s.add_data_point(mlm, ("my_survey", "z_bin_tracer_1", "mass_bin_tracer_1"), 1) + s.add_data_point(mlm, ("my_survey", "z_bin_tracer_1", "mass_bin_tracer_2"), 1) + s.add_data_point(mlm, ("not_my_survey", "z_bin_tracer_2", "mass_bin_tracer_1"), 1) + s.add_data_point(mlm, ("not_my_survey", "z_bin_tracer_2", "mass_bin_tracer_2"), 1) return s diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index ca6bd1af1..ff932e60c 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -19,7 +19,7 @@ def test_create_binned_number_counts(): integrator.integrate.return_value = 1.0 bnc = BinnedClusterNumberCounts(ClusterProperty.NONE, "Test", integrator) assert bnc is not None - assert bnc.properties == ClusterProperty.NONE + assert bnc.cluster_properties == ClusterProperty.NONE assert bnc.survey_name == "Test" assert bnc.systematics == [] assert bnc.theory_vector is None @@ -28,8 +28,8 @@ def test_create_binned_number_counts(): bnc = BinnedClusterNumberCounts( (ClusterProperty.COUNTS | ClusterProperty.MASS), "Test", integrator ) - assert ClusterProperty.COUNTS in bnc.properties - assert ClusterProperty.MASS in bnc.properties + assert ClusterProperty.COUNTS in bnc.cluster_properties + assert ClusterProperty.MASS in bnc.cluster_properties systematics = [SourceSystematic("mock_systematic")] bnc = BinnedClusterNumberCounts( @@ -61,25 +61,25 @@ def test_read(cluster_sacc_data: sacc.Sacc): bnc = BinnedClusterNumberCounts(ClusterProperty.COUNTS, "my_survey", integrator) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 - assert len(bnc.bin_limits) == 4 + assert len(bnc.bin_edges) == 2 assert len(bnc.data_vector) == 2 assert len(bnc.sacc_indices) == 2 bnc = BinnedClusterNumberCounts(ClusterProperty.MASS, "my_survey", integrator) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 - assert len(bnc.bin_limits) == 4 - assert len(bnc.data_vector) == 4 - assert len(bnc.sacc_indices) == 4 + assert len(bnc.bin_edges) == 2 + assert len(bnc.data_vector) == 2 + assert len(bnc.sacc_indices) == 2 bnc = BinnedClusterNumberCounts( (ClusterProperty.COUNTS | ClusterProperty.MASS), "my_survey", integrator ) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 - assert len(bnc.bin_limits) == 4 - assert len(bnc.data_vector) == 6 - assert len(bnc.sacc_indices) == 6 + assert len(bnc.bin_edges) == 2 + assert len(bnc.data_vector) == 4 + assert len(bnc.sacc_indices) == 4 def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): @@ -99,13 +99,13 @@ def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None - assert len(tv) == 4 + assert len(tv) == 2 bnc = BinnedClusterNumberCounts(ClusterProperty.MASS, "my_survey", integrator) bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None - assert len(tv) == 4 + assert len(tv) == 2 bnc = BinnedClusterNumberCounts( (ClusterProperty.COUNTS | ClusterProperty.MASS), "my_survey", integrator @@ -113,4 +113,4 @@ def test_compute_theory_vector(cluster_sacc_data: sacc.Sacc): bnc.read(cluster_sacc_data) tv = bnc.compute_theory_vector(tools) assert tv is not None - assert len(tv) == 8 + assert len(tv) == 4 diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index 5049b13de..3a20818f0 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -1,36 +1,15 @@ """Tests for the cluster abundance data module.""" import pytest -import numpy as np import sacc from firecrown.models.cluster.abundance_data import AbundanceData from firecrown.models.cluster.properties import ClusterProperty -def test_create_abundance_data_no_survey(): - with pytest.raises( - ValueError, match="The SACC file does not contain the SurveyTracer" - ): - _ = AbundanceData(sacc.Sacc(), "survey", ClusterProperty.NONE) - - -def test_create_abundance_data_wrong_tracer(): - s = sacc.Sacc() - s.add_tracer("bin_richness", "test", 0.1, 0.2) - with pytest.raises(ValueError, match="The SACC tracer test is not a SurveyTracer"): - _ = AbundanceData(s, "test", ClusterProperty.NONE) - - def test_create_abundance_data(): s = sacc.Sacc() - s.add_tracer("survey", "mock_survey", 4000) - ad = AbundanceData( - s, "mock_survey", (ClusterProperty.COUNTS | ClusterProperty.MASS) - ) + ad = AbundanceData(s, 2) - assert ClusterProperty.COUNTS in ad.properties - assert ClusterProperty.MASS in ad.properties - assert ad.survey_nm == "mock_survey" - assert ad.survey_tracer.sky_area == 4000 + assert ad.bin_dimensions == 2 # pylint: disable=protected-access assert ad._mass_index == 2 # pylint: disable=protected-access @@ -39,63 +18,95 @@ def test_create_abundance_data(): assert ad._survey_index == 0 -def test_validate_tracers(): - s = sacc.Sacc() - s.add_tracer("survey", "mock_survey", 4000) - s.add_tracer("bin_z", "my_tracer", 0, 2) - # pylint: disable=no-member - s.add_data_point( - sacc.standard_types.cluster_counts, ("mock_survey", "my_tracer"), 1 - ) - ad = AbundanceData(s, "mock_survey", ClusterProperty.NONE) - # pylint: disable=no-member - tracer_combs = np.array( - s.get_tracer_combinations(sacc.standard_types.cluster_mean_log_mass) - ) - # pylint: disable=no-member +def test_get_survey_tracer_missing_survey_name(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 2) with pytest.raises( ValueError, - match="The SACC file does not contain any tracers for the" - + f" {sacc.standard_types.cluster_mean_log_mass} data type", + match="The SACC file does not contain the SurveyTracer the_black_lodge.", ): - # pylint: disable=no-member - ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_mean_log_mass) + ad.get_survey_tracer("the_black_lodge") + + +def test_get_survey_tracer_wrong_tracer_type(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 2) + with pytest.raises( + ValueError, + match="The SACC tracer z_bin_tracer_1 is not a SurveyTracer.", + ): + ad.get_survey_tracer("z_bin_tracer_1") + + +def test_get_survey_tracer_returns_survey_tracer(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 2) + survey = ad.get_survey_tracer("my_survey") + assert survey is not None + assert isinstance(survey, sacc.tracers.SurveyTracer) - # pylint: disable=no-member - tracer_combs = np.array( - s.get_tracer_combinations(sacc.standard_types.cluster_counts) + +def test_get_bin_edges_cluster_counts(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 2) + bins = ad.get_bin_edges("my_survey", ClusterProperty.COUNTS) + assert len(bins) == 2 + + +def test_get_bin_edges_cluster_mass(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 2) + bins = ad.get_bin_edges("my_survey", ClusterProperty.MASS) + assert len(bins) == 2 + + +def test_get_bin_edges_counts_and_mass(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 2) + bins = ad.get_bin_edges( + "my_survey", (ClusterProperty.MASS | ClusterProperty.COUNTS) + ) + assert len(bins) == 2 + + +def test_get_bin_edges_not_implemented_cluster_property_throws( + cluster_sacc_data: sacc.Sacc, +): + ad = AbundanceData(cluster_sacc_data, 2) + with pytest.raises(NotImplementedError): + ad.get_bin_edges("my_survey", ClusterProperty.SHEAR) + + +def test_observed_data_and_indices_by_survey_cluster_counts( + cluster_sacc_data: sacc.Sacc, +): + ad = AbundanceData(cluster_sacc_data, 2) + data, indices = ad.get_observed_data_and_indices_by_survey( + "my_survey", ClusterProperty.COUNTS ) - with pytest.raises(ValueError, match="The SACC file must contain 3 tracers"): - # pylint: disable=no-member - ad.validate_tracers(tracer_combs, sacc.standard_types.cluster_counts) - - -def test_filtered_tracers(cluster_sacc_data): - ad = AbundanceData(cluster_sacc_data, "my_survey", ClusterProperty.NONE) - # pylint: disable=no-member - cc = sacc.standard_types.cluster_counts - filtered_tracers, survey_mask = ad.get_filtered_tracers(cc) - my_tracers = [ - ("my_survey", "my_tracer1", "my_other_tracer1"), - ("my_survey", "my_tracer1", "my_other_tracer2"), - ] - assert (filtered_tracers == my_tracers).all() - assert (survey_mask == [True, True, False]).all() - - -def test_get_data_and_indices(cluster_sacc_data): - ad = AbundanceData(cluster_sacc_data, "my_survey", ClusterProperty.NONE) - # pylint: disable=no-member - cc = sacc.standard_types.cluster_counts - data, indices = ad.get_data_and_indices(cc) - - assert data == [1, 1] - assert indices == [0, 1] - - -def test_get_bin_limits(cluster_sacc_data): - ad = AbundanceData(cluster_sacc_data, "my_survey", ClusterProperty.NONE) - # pylint: disable=no-member - cc = sacc.standard_types.cluster_counts - limits = ad.get_bin_limits(cc) - assert limits == [[(0, 2), (0, 2)], [(0, 2), (2, 4)]] + assert len(data) == 2 + assert len(indices) == 2 + + +def test_observed_data_and_indices_by_survey_cluster_mass( + cluster_sacc_data: sacc.Sacc, +): + ad = AbundanceData(cluster_sacc_data, 2) + data, indices = ad.get_observed_data_and_indices_by_survey( + "my_survey", ClusterProperty.MASS + ) + assert len(data) == 2 + assert len(indices) == 2 + + +def test_observed_data_and_indices_by_survey_cluster_counts_and_mass( + cluster_sacc_data: sacc.Sacc, +): + ad = AbundanceData(cluster_sacc_data, 2) + data, indices = ad.get_observed_data_and_indices_by_survey( + "my_survey", ClusterProperty.MASS | ClusterProperty.COUNTS + ) + assert len(data) == 4 + assert len(indices) == 4 + + +def test_observed_data_and_indices_by_survey_not_implemented_throws( + cluster_sacc_data: sacc.Sacc, +): + ad = AbundanceData(cluster_sacc_data, 2) + with pytest.raises(NotImplementedError): + ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.SHEAR) diff --git a/tests/test_cluster_integrators.py b/tests/test_cluster_integrators.py index 93f710ee7..481aed993 100644 --- a/tests/test_cluster_integrators.py +++ b/tests/test_cluster_integrators.py @@ -8,7 +8,7 @@ from firecrown.models.cluster.integrator.scipy_integrator import ScipyIntegrator from firecrown.models.cluster.integrator.numcosmo_integrator import NumCosmoIntegrator from firecrown.models.cluster.kernel import KernelType, Kernel -from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand +from firecrown.models.cluster.abundance import ClusterAbundance @pytest.fixture(name="integrator", params=[ScipyIntegrator, NumCosmoIntegrator]) From 91cf56beaef5b6016d026aa5e3b9b5aacc4e4929 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 1 Dec 2023 13:31:12 -0800 Subject: [PATCH 67/80] Forgot to update tests for ClusterAbundance --- tests/test_cluster_abundance.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index 0c21c4a97..eb564e977 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -3,7 +3,6 @@ import pytest import pyccl import numpy as np -from firecrown.parameters import ParamsMap from firecrown.models.cluster.abundance import ClusterAbundance from firecrown.models.cluster.kernel import Kernel, KernelType from firecrown.models.cluster.properties import ClusterProperty @@ -42,10 +41,9 @@ def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): # pylint: disable=protected-access assert cl_abundance._hmf_cache == {} - pmap = ParamsMap({}) cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, pmap) + cl_abundance.update_ingredients(cosmo) assert cl_abundance.cosmo is not None assert cl_abundance.cosmo == cosmo # pylint: disable=protected-access @@ -103,7 +101,7 @@ def test_cluster_add_kernel(cl_abundance: ClusterAbundance): def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) result = cl_abundance.comoving_volume(np.linspace(0.1, 1, 10), 360**2) assert isinstance(result, np.ndarray) @@ -115,7 +113,7 @@ def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): @pytest.mark.regression def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) result = cl_abundance.mass_function(np.linspace(13, 17, 5), np.linspace(0.1, 1, 5)) assert isinstance(result, np.ndarray) @@ -127,7 +125,7 @@ def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): @pytest.mark.regression def test_abundance_get_integrand(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) mk = Mock( spec=Kernel, kernel_type=KernelType.MASS, @@ -157,7 +155,7 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): @pytest.mark.regression def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) mk = Mock( spec=Kernel, kernel_type=KernelType.MASS, @@ -187,7 +185,7 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): @pytest.mark.regression def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) mk = Mock( spec=Kernel, kernel_type=KernelType.MASS, @@ -217,7 +215,7 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): @pytest.mark.regression def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) mk = Mock( spec=Kernel, kernel_type=KernelType.MASS, @@ -250,7 +248,7 @@ def test_abundance_get_integrand_avg_not_implemented_throws( cl_abundance: ClusterAbundance, ): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo, ParamsMap()) + cl_abundance.update_ingredients(cosmo) mk = Mock( spec=Kernel, kernel_type=KernelType.MASS, From 61c2a84da1d2bec571080fc1e204042425e2643d Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Fri, 1 Dec 2023 15:18:05 -0800 Subject: [PATCH 68/80] Python 3.9, 3.10 tests are failing on build server, while 3.11 passing locally. Switching 3.11 syntax for older syntax. Updating cluster abundance tests to be more better. --- .../statistic/binned_cluster_number_counts.py | 5 +- firecrown/models/cluster/abundance.py | 14 +- firecrown/models/cluster/abundance_data.py | 4 +- tests/test_cluster_abundance.py | 232 +++++++++--------- 4 files changed, 123 insertions(+), 132 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 676228614..33dad3b09 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -8,7 +8,8 @@ import sacc import numpy as np from firecrown.models.cluster.integrator.integrator import Integrator -from firecrown.models.cluster.abundance_data import AbundanceData, SaccBin +from firecrown.models.cluster.abundance_data import AbundanceData +from firecrown.models.cluster.binning import SaccBin from firecrown.models.cluster.properties import ClusterProperty from firecrown.likelihood.gauss_family.statistic.statistic import ( Statistic, @@ -106,7 +107,7 @@ def get_binned_cluster_property( cluster_masses = [] for bin_edge, counts in zip(self.bin_edges, cluster_counts): integrand = tools.cluster_abundance.get_integrand( - average=cluster_properties + average_properties=cluster_properties ) self.integrator.set_integration_bounds( tools.cluster_abundance, diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index d63d0ba36..bf2321762 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -128,7 +128,7 @@ def mass_function( return np.asarray(return_vals, dtype=np.float64) def get_integrand( - self, *, average: Optional[ClusterProperty] = None + self, *, average_properties: Optional[ClusterProperty] = None ) -> AbundanceIntegrand: """Returns a callable that evaluates the complete integrand.""" @@ -149,18 +149,18 @@ def integrand( mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits ) - if average is None: + if average_properties is None: return integrand - for prop in ClusterProperty: - if not prop & average: + for cluster_prop in ClusterProperty: + if not cluster_prop & average_properties: continue - if prop == ClusterProperty.MASS: + if cluster_prop == ClusterProperty.MASS: integrand *= mass - elif prop == ClusterProperty.REDSHIFT: + elif cluster_prop == ClusterProperty.REDSHIFT: integrand *= z else: - raise NotImplementedError(f"Average for {prop}.") + raise NotImplementedError(f"Average for {cluster_prop}.") return integrand diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 2fc62e48e..b7126c6b9 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -59,7 +59,7 @@ def get_observed_data_and_indices_by_survey( sacc_indices = [] for cluster_property in ClusterProperty: - if cluster_property not in properties: + if not cluster_property & properties: continue if cluster_property == ClusterProperty.COUNTS: @@ -113,7 +113,7 @@ def get_bin_edges( bins = [] for cluster_property in ClusterProperty: - if cluster_property not in properties: + if not cluster_property & properties: continue if cluster_property == ClusterProperty.COUNTS: diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index eb564e977..efa3a9e3d 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -8,102 +8,115 @@ from firecrown.models.cluster.properties import ClusterProperty -@pytest.fixture(name="cl_abundance") -def fixture_cl_abundance(): +@pytest.fixture(name="cluster_abundance") +def fixture_cluster_abundance(): """Test fixture that represents an assembled cluster abundance class.""" hmf = pyccl.halos.MassFuncBocquet16() ca = ClusterAbundance(13, 17, 0, 2, hmf) return ca -def test_cluster_abundance_init(cl_abundance: ClusterAbundance): - assert cl_abundance is not None - assert cl_abundance.cosmo is None - assert isinstance(cl_abundance.halo_mass_function, pyccl.halos.MassFuncBocquet16) - assert cl_abundance.min_mass == 13.0 - assert cl_abundance.max_mass == 17.0 - assert cl_abundance.min_z == 0.0 - assert cl_abundance.max_z == 2.0 - assert len(cl_abundance.kernels) == 0 - - -def test_cluster_update_ingredients(cl_abundance: ClusterAbundance): +@pytest.fixture(name="dirac_delta_kernel") +def fixture_dirac_delta_kernel(): mk = Mock( spec=Kernel, - kernel_type=KernelType.MASS_PROXY, + kernel_type=KernelType.MASS, is_dirac_delta=True, has_analytic_sln=False, ) mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) - - assert cl_abundance.cosmo is None - # pylint: disable=protected-access - assert cl_abundance._hmf_cache == {} - - cosmo = pyccl.CosmologyVanillaLCDM() - - cl_abundance.update_ingredients(cosmo) - assert cl_abundance.cosmo is not None - assert cl_abundance.cosmo == cosmo - # pylint: disable=protected-access - assert cl_abundance._hmf_cache == {} - - cl_abundance.update_ingredients(None) - assert cl_abundance.cosmo is None - cl_abundance.update_ingredients(cosmo) - assert cl_abundance.cosmo is not None - assert cl_abundance.cosmo == cosmo + return mk -def test_cluster_add_kernel(cl_abundance: ClusterAbundance): - assert len(cl_abundance.kernels) == 0 - +@pytest.fixture(name="integrable_kernel") +def fixture_integrable_kernel(): mk = Mock( spec=Kernel, - kernel_type=KernelType.MASS, + kernel_type=KernelType.MASS_PROXY, is_dirac_delta=False, has_analytic_sln=False, ) mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) - assert len(cl_abundance.kernels) == 1 - assert isinstance(cl_abundance.kernels[0], Kernel) - assert cl_abundance.kernels[0].kernel_type == KernelType.MASS + return mk - assert len(cl_abundance.analytic_kernels) == 0 - assert len(cl_abundance.dirac_delta_kernels) == 0 - assert len(cl_abundance.integrable_kernels) == 1 +@pytest.fixture(name="analytic_sln_kernel") +def fixture_analytic_sln_solved_kernel(): mk = Mock( spec=Kernel, kernel_type=KernelType.MASS, - is_dirac_delta=True, - has_analytic_sln=False, + is_dirac_delta=False, + has_analytic_sln=True, ) mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) - assert len(cl_abundance.analytic_kernels) == 0 - assert len(cl_abundance.dirac_delta_kernels) == 1 - assert len(cl_abundance.integrable_kernels) == 1 + return mk - mk = Mock( - spec=Kernel, - kernel_type=KernelType.MASS, - is_dirac_delta=False, - has_analytic_sln=True, + +def test_cluster_abundance_init(cluster_abundance: ClusterAbundance): + assert cluster_abundance is not None + assert cluster_abundance.cosmo is None + # pylint: disable=protected-access + assert cluster_abundance._hmf_cache == {} + assert isinstance( + cluster_abundance.halo_mass_function, pyccl.halos.MassFuncBocquet16 ) - cl_abundance.add_kernel(mk) - assert len(cl_abundance.analytic_kernels) == 1 - assert len(cl_abundance.dirac_delta_kernels) == 1 - assert len(cl_abundance.integrable_kernels) == 1 + assert cluster_abundance.min_mass == 13.0 + assert cluster_abundance.max_mass == 17.0 + assert cluster_abundance.min_z == 0.0 + assert cluster_abundance.max_z == 2.0 + assert len(cluster_abundance.kernels) == 0 + + +def test_cluster_update_ingredients(cluster_abundance: ClusterAbundance): + cosmo = pyccl.CosmologyVanillaLCDM() + cluster_abundance.update_ingredients(cosmo) + assert cluster_abundance.cosmo is not None + assert cluster_abundance.cosmo == cosmo + # pylint: disable=protected-access + assert cluster_abundance._hmf_cache == {} + + +def test_cluster_add_integrable_kernel( + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel +): + cluster_abundance.add_kernel(integrable_kernel) + assert len(cluster_abundance.kernels) == 1 + assert isinstance(cluster_abundance.kernels[0], Kernel) + + assert len(cluster_abundance.analytic_kernels) == 0 + assert len(cluster_abundance.dirac_delta_kernels) == 0 + assert len(cluster_abundance.integrable_kernels) == 1 + + +def test_cluster_add_dirac_delta_kernel( + cluster_abundance: ClusterAbundance, dirac_delta_kernel: Kernel +): + cluster_abundance.add_kernel(dirac_delta_kernel) + assert len(cluster_abundance.kernels) == 1 + assert isinstance(cluster_abundance.kernels[0], Kernel) + + assert len(cluster_abundance.analytic_kernels) == 0 + assert len(cluster_abundance.dirac_delta_kernels) == 1 + assert len(cluster_abundance.integrable_kernels) == 0 + + +def test_cluster_add_analytic_sln_kernel( + cluster_abundance: ClusterAbundance, analytic_sln_kernel: Kernel +): + cluster_abundance.add_kernel(analytic_sln_kernel) + assert len(cluster_abundance.kernels) == 1 + assert isinstance(cluster_abundance.kernels[0], Kernel) + + assert len(cluster_abundance.analytic_kernels) == 1 + assert len(cluster_abundance.dirac_delta_kernels) == 0 + assert len(cluster_abundance.integrable_kernels) == 0 -def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): +def test_abundance_comoving_returns_value(cluster_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) + cluster_abundance.update_ingredients(cosmo) - result = cl_abundance.comoving_volume(np.linspace(0.1, 1, 10), 360**2) + result = cluster_abundance.comoving_volume(np.linspace(0.1, 1, 10), 360**2) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) assert len(result) == 10 @@ -111,11 +124,13 @@ def test_abundance_comoving_vol_accepts_array(cl_abundance: ClusterAbundance): @pytest.mark.regression -def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): +def test_abundance_massfunc_returns_value(cluster_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) + cluster_abundance.update_ingredients(cosmo) - result = cl_abundance.mass_function(np.linspace(13, 17, 5), np.linspace(0.1, 1, 5)) + result = cluster_abundance.mass_function( + np.linspace(13, 17, 5), np.linspace(0.1, 1, 5) + ) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) assert len(result) == 5 @@ -123,17 +138,12 @@ def test_abundance_massfunc_accepts_array(cl_abundance: ClusterAbundance): @pytest.mark.regression -def test_abundance_get_integrand(cl_abundance: ClusterAbundance): +def test_abundance_get_integrand( + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel +): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) - mk = Mock( - spec=Kernel, - kernel_type=KernelType.MASS, - is_dirac_delta=False, - has_analytic_sln=False, - ) - mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) + cluster_abundance.update_ingredients(cosmo) + cluster_abundance.add_kernel(integrable_kernel) sky_area = 439.78986 mass = np.linspace(13, 17, 5) @@ -143,7 +153,7 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): mass_proxy_limits = (0, 5) z_proxy_limits = (0, 1) - integrand = cl_abundance.get_integrand() + integrand = cluster_abundance.get_integrand() assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits @@ -153,17 +163,12 @@ def test_abundance_get_integrand(cl_abundance: ClusterAbundance): @pytest.mark.regression -def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): +def test_abundance_get_integrand_avg_mass( + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel +): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) - mk = Mock( - spec=Kernel, - kernel_type=KernelType.MASS, - is_dirac_delta=False, - has_analytic_sln=False, - ) - mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) + cluster_abundance.update_ingredients(cosmo) + cluster_abundance.add_kernel(integrable_kernel) sky_area = 439.78986 mass = np.linspace(13, 17, 5) @@ -173,7 +178,7 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): mass_proxy_limits = (0, 5) z_proxy_limits = (0, 1) - integrand = cl_abundance.get_integrand(average=ClusterProperty.MASS) + integrand = cluster_abundance.get_integrand(average_properties=ClusterProperty.MASS) assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits @@ -183,17 +188,12 @@ def test_abundance_get_integrand_avg_mass(cl_abundance: ClusterAbundance): @pytest.mark.regression -def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): +def test_abundance_get_integrand_avg_redshift( + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel +): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) - mk = Mock( - spec=Kernel, - kernel_type=KernelType.MASS, - is_dirac_delta=False, - has_analytic_sln=False, - ) - mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) + cluster_abundance.update_ingredients(cosmo) + cluster_abundance.add_kernel(integrable_kernel) sky_area = 439.78986 mass = np.linspace(13, 17, 5) @@ -203,7 +203,9 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): mass_proxy_limits = (0, 5) z_proxy_limits = (0, 1) - integrand = cl_abundance.get_integrand(average=ClusterProperty.REDSHIFT) + integrand = cluster_abundance.get_integrand( + average_properties=ClusterProperty.REDSHIFT + ) assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits @@ -213,17 +215,12 @@ def test_abundance_get_integrand_avg_redshift(cl_abundance: ClusterAbundance): @pytest.mark.regression -def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbundance): +def test_abundance_get_integrand_avg_mass_and_redshift( + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel +): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) - mk = Mock( - spec=Kernel, - kernel_type=KernelType.MASS, - is_dirac_delta=False, - has_analytic_sln=False, - ) - mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) + cluster_abundance.update_ingredients(cosmo) + cluster_abundance.add_kernel(integrable_kernel) sky_area = 439.78986 mass = np.linspace(13, 17, 5) @@ -234,7 +231,7 @@ def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbun z_proxy_limits = (0, 1) average_properties = ClusterProperty.MASS | ClusterProperty.REDSHIFT - integrand = cl_abundance.get_integrand(average=average_properties) + integrand = cluster_abundance.get_integrand(average_properties=average_properties) assert callable(integrand) result = integrand( mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits @@ -245,18 +242,11 @@ def test_abundance_get_integrand_avg_mass_and_redshift(cl_abundance: ClusterAbun @pytest.mark.regression def test_abundance_get_integrand_avg_not_implemented_throws( - cl_abundance: ClusterAbundance, + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel ): cosmo = pyccl.CosmologyVanillaLCDM() - cl_abundance.update_ingredients(cosmo) - mk = Mock( - spec=Kernel, - kernel_type=KernelType.MASS, - is_dirac_delta=False, - has_analytic_sln=False, - ) - mk.distribution.return_value = np.atleast_1d(1.0) - cl_abundance.add_kernel(mk) + cluster_abundance.update_ingredients(cosmo) + cluster_abundance.add_kernel(integrable_kernel) sky_area = 439.78986 mass = np.linspace(13, 17, 5) @@ -267,7 +257,7 @@ def test_abundance_get_integrand_avg_not_implemented_throws( z_proxy_limits = (0, 1) average_properties = ClusterProperty.SHEAR - integrand = cl_abundance.get_integrand(average=average_properties) + integrand = cluster_abundance.get_integrand(average_properties=average_properties) assert callable(integrand) with pytest.raises(NotImplementedError): _ = integrand( From 11e023e869fb77b255b95688f70b28ecf9a7f3b2 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Sun, 3 Dec 2023 00:00:51 -0800 Subject: [PATCH 69/80] Improving tests. --- tests/test_cluster_abundance.py | 82 +++++++++------------------------ 1 file changed, 22 insertions(+), 60 deletions(-) diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index efa3a9e3d..b5c1192ff 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -52,6 +52,18 @@ def fixture_analytic_sln_solved_kernel(): return mk +@pytest.fixture(name="integrand_args") +def fixture_integrand_args(): + sky_area = 439.78986 + mass = np.linspace(13, 17, 5) + z = np.linspace(0, 1, 5) + mass_proxy = np.linspace(0, 5, 5) + z_proxy = np.linspace(0, 1, 5) + mass_proxy_limits = (0, 5) + z_proxy_limits = (0, 1) + return (mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits) + + def test_cluster_abundance_init(cluster_abundance: ClusterAbundance): assert cluster_abundance is not None assert cluster_abundance.cosmo is None @@ -139,127 +151,77 @@ def test_abundance_massfunc_returns_value(cluster_abundance: ClusterAbundance): @pytest.mark.regression def test_abundance_get_integrand( - cluster_abundance: ClusterAbundance, integrable_kernel: Kernel + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): cosmo = pyccl.CosmologyVanillaLCDM() cluster_abundance.update_ingredients(cosmo) cluster_abundance.add_kernel(integrable_kernel) - sky_area = 439.78986 - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) - integrand = cluster_abundance.get_integrand() assert callable(integrand) - result = integrand( - mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits - ) + result = integrand(*integrand_args) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) @pytest.mark.regression def test_abundance_get_integrand_avg_mass( - cluster_abundance: ClusterAbundance, integrable_kernel: Kernel + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): cosmo = pyccl.CosmologyVanillaLCDM() cluster_abundance.update_ingredients(cosmo) cluster_abundance.add_kernel(integrable_kernel) - sky_area = 439.78986 - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) - integrand = cluster_abundance.get_integrand(average_properties=ClusterProperty.MASS) assert callable(integrand) - result = integrand( - mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits - ) + result = integrand(*integrand_args) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) @pytest.mark.regression def test_abundance_get_integrand_avg_redshift( - cluster_abundance: ClusterAbundance, integrable_kernel: Kernel + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): cosmo = pyccl.CosmologyVanillaLCDM() cluster_abundance.update_ingredients(cosmo) cluster_abundance.add_kernel(integrable_kernel) - sky_area = 439.78986 - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) - integrand = cluster_abundance.get_integrand( average_properties=ClusterProperty.REDSHIFT ) assert callable(integrand) - result = integrand( - mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits - ) + result = integrand(*integrand_args) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) @pytest.mark.regression def test_abundance_get_integrand_avg_mass_and_redshift( - cluster_abundance: ClusterAbundance, integrable_kernel: Kernel + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): cosmo = pyccl.CosmologyVanillaLCDM() cluster_abundance.update_ingredients(cosmo) cluster_abundance.add_kernel(integrable_kernel) - sky_area = 439.78986 - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) - average_properties = ClusterProperty.MASS | ClusterProperty.REDSHIFT integrand = cluster_abundance.get_integrand(average_properties=average_properties) assert callable(integrand) - result = integrand( - mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits - ) + result = integrand(*integrand_args) assert isinstance(result, np.ndarray) assert np.issubdtype(result.dtype, np.float64) @pytest.mark.regression def test_abundance_get_integrand_avg_not_implemented_throws( - cluster_abundance: ClusterAbundance, integrable_kernel: Kernel + cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): cosmo = pyccl.CosmologyVanillaLCDM() cluster_abundance.update_ingredients(cosmo) cluster_abundance.add_kernel(integrable_kernel) - sky_area = 439.78986 - mass = np.linspace(13, 17, 5) - z = np.linspace(0, 1, 5) - mass_proxy = np.linspace(0, 5, 5) - z_proxy = np.linspace(0, 1, 5) - mass_proxy_limits = (0, 5) - z_proxy_limits = (0, 1) - average_properties = ClusterProperty.SHEAR integrand = cluster_abundance.get_integrand(average_properties=average_properties) assert callable(integrand) with pytest.raises(NotImplementedError): - _ = integrand( - mass, z, sky_area, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits - ) + _ = integrand(*integrand_args) From aa4c887bd817f13c55b761844cce814af065a916 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Sun, 3 Dec 2023 01:58:26 -0800 Subject: [PATCH 70/80] Updating code to latest numcosmo version. --- examples/des_y1_3x2pt/numcosmo_run.py | 10 +++++----- examples/des_y1_3x2pt/numcosmo_run_PT.py | 2 +- examples/srd_sn/numcosmo_run.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/des_y1_3x2pt/numcosmo_run.py b/examples/des_y1_3x2pt/numcosmo_run.py index 1dd3e6098..bfcd60c82 100755 --- a/examples/des_y1_3x2pt/numcosmo_run.py +++ b/examples/des_y1_3x2pt/numcosmo_run.py @@ -129,7 +129,7 @@ def setup_likelihood( def setup_fit(likelihood: Ncm.Likelihood, mset: Ncm.MSet) -> Ncm.Fit: """Setup the fit object.""" - fit = Ncm.Fit.new( + fit = Ncm.Fit.factory( Ncm.FitType.NLOPT, "ln-neldermead", likelihood, @@ -151,7 +151,7 @@ def setup_fit_all() -> Ncm.Fit: return fit -def run_test(): +def run_test() -> None: """Run the fit.""" fit = setup_fit_all() @@ -162,7 +162,7 @@ def run_test(): fit.run_restart(Ncm.FitRunMsgs.FULL, 1.0e-3, 0.0, None, None) -def run_compute_best_fit(): +def run_compute_best_fit() -> None: """Run the fit.""" fit = setup_fit_all() @@ -178,7 +178,7 @@ def run_compute_best_fit(): fit.run_restart(Ncm.FitRunMsgs.FULL, 1.0e-3, 0.0, None, None) -def run_apes_sampler(ssize: int): +def run_apes_sampler(ssize: int) -> None: """Run the fit.""" fit = setup_fit_all() @@ -192,7 +192,7 @@ def run_apes_sampler(ssize: int): nwalkers = mset.fparam_len * 100 esmcmc = create_esmcmc( - fit.lh, mset, "des_y1_3x2pt_apes", nwalkers=nwalkers, nthreads=1 + fit.likelihood, mset, "des_y1_3x2pt_apes", nwalkers=nwalkers, nthreads=1 ) esmcmc.start_run() diff --git a/examples/des_y1_3x2pt/numcosmo_run_PT.py b/examples/des_y1_3x2pt/numcosmo_run_PT.py index 4935b00c0..a73821539 100755 --- a/examples/des_y1_3x2pt/numcosmo_run_PT.py +++ b/examples/des_y1_3x2pt/numcosmo_run_PT.py @@ -77,7 +77,7 @@ lh.priors_add_gauss_param_name(mset, "NcFirecrownPT:src0_delta_z", -0.001, 0.016) lh.priors_add_gauss_param_name(mset, "NcFirecrownPT:lens0_delta_z", +0.001, 0.008) -fit = Ncm.Fit.new( +fit = Ncm.Fit.factory( Ncm.FitType.NLOPT, "ln-neldermead", lh, diff --git a/examples/srd_sn/numcosmo_run.py b/examples/srd_sn/numcosmo_run.py index 6577f0c34..2375618bd 100755 --- a/examples/srd_sn/numcosmo_run.py +++ b/examples/srd_sn/numcosmo_run.py @@ -81,7 +81,7 @@ lh = Ncm.Likelihood(dataset=dset) -fit = Ncm.Fit.new( +fit = Ncm.Fit.factory( Ncm.FitType.NLOPT, "ln-neldermead", lh, From 1b51c3c3cea7f57f8a20d8095ac2a5d367bb5c87 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Sun, 3 Dec 2023 02:12:41 -0800 Subject: [PATCH 71/80] Missed places to upgrade numcosmo references. --- .../generate_rich_mean_mass_sacc_data.py | 17 ++++++----------- examples/des_y1_3x2pt/numcosmo_run.py | 2 +- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py b/examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py index 43c2bc78b..5666faf4f 100755 --- a/examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py +++ b/examples/cluster_number_counts/generate_rich_mean_mass_sacc_data.py @@ -19,11 +19,11 @@ from astropy.io import fits from scipy import stats - +from typing import Any import sacc -def generate_sacc_file(): +def generate_sacc_file() -> Any: """Generate a SACC file for cluster number counts.""" H0 = 71.0 @@ -40,8 +40,7 @@ def generate_sacc_file(): cosmo.add_submodel(prim) dist = Nc.Distance.new(2.0) - - tf = Nc.TransferFunc.new_from_name("NcTransferFuncEH") + tf = Nc.TransferFuncEH.new() psml = Nc.PowspecMLTransfer.new(tf) @@ -74,13 +73,9 @@ def generate_sacc_file(): zu = 0.65 # NumCosmo proxy model based on arxiv 1904.07524v2 - cluster_z = Nc.ClusterRedshift.new_from_name( - f"NcClusterRedshiftNodist{{'z-min': <{zl:22.15e}>, 'z-max':<{zu:22.15e}>}}" - ) - - cluster_m = Nc.ClusterMass.new_from_name( - f"NcClusterMassAscaso{{'M0':<{3.0e14 / 0.71:22.15e}>,'z0':<0.6>, " - f"'lnRichness-min':<{lnRl:22.15e}>, 'lnRichness-max':<{lnRu:22.15e}>}}" + cluster_z = Nc.ClusterRedshiftNodist(z_max=zu, z_min=zl) + cluster_m = Nc.ClusterMassAscaso( + M0=3.0e14 / 0.71, z0=0.6, lnRichness_min=lnRl, lnRichness_max=lnRu ) cluster_m.param_set_by_name("mup0", 3.19) cluster_m.param_set_by_name("mup1", 2 / np.log(10)) diff --git a/examples/des_y1_3x2pt/numcosmo_run.py b/examples/des_y1_3x2pt/numcosmo_run.py index bfcd60c82..ef6d029e4 100755 --- a/examples/des_y1_3x2pt/numcosmo_run.py +++ b/examples/des_y1_3x2pt/numcosmo_run.py @@ -192,7 +192,7 @@ def run_apes_sampler(ssize: int) -> None: nwalkers = mset.fparam_len * 100 esmcmc = create_esmcmc( - fit.likelihood, mset, "des_y1_3x2pt_apes", nwalkers=nwalkers, nthreads=1 + fit.props.likelihood, mset, "des_y1_3x2pt_apes", nwalkers=nwalkers, nthreads=1 ) esmcmc.start_run() From 3319fa8c0143e1f9adf471d1675c05bc2fdf0fd1 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 4 Dec 2023 15:36:42 -0800 Subject: [PATCH 72/80] Changes to binning.py --- .../statistic/binned_cluster_number_counts.py | 2 +- firecrown/models/cluster/abundance.py | 2 +- firecrown/models/cluster/abundance_data.py | 10 +++-- firecrown/models/cluster/binning.py | 35 +++++++++++------- setup.cfg | 3 -- tests/test_cluster_binning.py | 11 ++++++ tests/test_cluster_data.py | 37 +++++++++++++++++++ 7 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 tests/test_cluster_binning.py diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 33dad3b09..bb2fbdaff 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -78,7 +78,7 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_counts = self.get_binned_cluster_counts(tools) for cl_property in ClusterProperty: - if not cl_property & self.cluster_properties: + if not (cl_property & self.cluster_properties): continue if cl_property == ClusterProperty.COUNTS: diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index bf2321762..9d74677cb 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -153,7 +153,7 @@ def integrand( return integrand for cluster_prop in ClusterProperty: - if not cluster_prop & average_properties: + if not (cluster_prop & average_properties): continue if cluster_prop == ClusterProperty.MASS: integrand *= mass diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index b7126c6b9..7da865d4a 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -59,7 +59,7 @@ def get_observed_data_and_indices_by_survey( sacc_indices = [] for cluster_property in ClusterProperty: - if not cluster_property & properties: + if not (cluster_property & properties): continue if cluster_property == ClusterProperty.COUNTS: @@ -130,9 +130,11 @@ def get_bin_edges( bin_combinations_for_survey = bin_combinations[my_survey_mask] for _, z_tracer, mass_tracer in bin_combinations_for_survey: - z_data: sacc.BaseTracer = self.sacc_data.get_tracer(z_tracer) - mass_data: sacc.BaseTracer = self.sacc_data.get_tracer(mass_tracer) - sacc_bin = SaccBin([z_data, mass_data], self.bin_dimensions) + z_data: sacc.tracers.BinZTracer = self.sacc_data.get_tracer(z_tracer) + mass_data: sacc.tracers.BinRichnessTracer = self.sacc_data.get_tracer( + mass_tracer + ) + sacc_bin = SaccBin([z_data, mass_data]) bins.append(sacc_bin) # Remove duplicates while preserving order (i.e. dont use set()) diff --git a/firecrown/models/cluster/binning.py b/firecrown/models/cluster/binning.py index a5b34343d..c761c618e 100644 --- a/firecrown/models/cluster/binning.py +++ b/firecrown/models/cluster/binning.py @@ -12,10 +12,15 @@ class NDimensionalBin(Generic[T], ABC): """Class which defines the interface for an N dimensional bin used in the cluster likelihood.""" - def __init__(self, bins: List[T], dimension: int): - self.bins = bins - self.dimension = dimension - assert len(bins) == dimension + def __init__(self, coordinate_bins: List[T]): + """_summary_ + + Args: + coordinate_bins (List[T]): _description_ + dimension (int): _description_ + """ + self.coordinate_bins = coordinate_bins + self.dimension = len(coordinate_bins) @property @abstractmethod @@ -37,16 +42,16 @@ def __repr__(self) -> str: class SaccBin(NDimensionalBin[sacc.BaseTracer]): """An implementation of the N dimensional bin using sacc tracers.""" - def __init__(self, bins: List[sacc.BaseTracer], dimension: int): - super().__init__(bins, dimension) + def __init__(self, bins: List[sacc.BaseTracer]): + super().__init__(bins) @property def z_edges(self) -> Tuple[float, float]: - return self.bins[0].lower, self.bins[0].upper + return self.coordinate_bins[0].lower, self.coordinate_bins[0].upper @property def mass_proxy_edges(self) -> Tuple[float, float]: - return self.bins[1].lower, self.bins[1].upper + return self.coordinate_bins[1].lower, self.coordinate_bins[1].upper def __eq__(self, other: object) -> bool: """Two bins are equal if they have the same lower/upper bound.""" @@ -56,16 +61,20 @@ def __eq__(self, other: object) -> bool: if self.dimension != other.dimension: return False - for i, my_bin in enumerate(self.bins): - other_bin = other.bins[i] - if my_bin.lower != other_bin.lower: + for i, my_bin in enumerate(self.coordinate_bins): + other_bin = [ + x for x in other.coordinate_bins if x.tracer_type == my_bin.tracer_type + ] + if len(other_bin) != 1: + return False + if my_bin.lower != other_bin[0].lower: return False - if my_bin.upper != other_bin.upper: + if my_bin.upper != other_bin[0].upper: return False return True def __hash__(self) -> int: """One bin's hash is determined by the dimension and lower/upper bound.""" - bin_bounds = [(bin.lower, bin.upper) for bin in self.bins] + bin_bounds = [(bin.lower, bin.upper) for bin in self.coordinate_bins] return hash((self.dimension, tuple(bin_bounds))) diff --git a/setup.cfg b/setup.cfg index 21a239d39..7115eb579 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,6 +55,3 @@ explicit_package_bases = True [mypy-firecrown.connector.cobaya.*] disallow_subclassing_any = False - -[mypy-tests.*] -ignore_errors = True \ No newline at end of file diff --git a/tests/test_cluster_binning.py b/tests/test_cluster_binning.py new file mode 100644 index 000000000..3e8ab1699 --- /dev/null +++ b/tests/test_cluster_binning.py @@ -0,0 +1,11 @@ +from firecrown.models.cluster.binning import SaccBin +import sacc + + +def test_create_sacc_bin_with_correct_dimension(): + tracer = sacc.tracers.SurveyTracer("", 1) + for i in range(10): + tracers = [tracer for _ in range(i)] + sb = SaccBin(tracers) + assert sb is not None + assert sb.dimension == i diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index 3a20818f0..5556dd92a 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -110,3 +110,40 @@ def test_observed_data_and_indices_by_survey_not_implemented_throws( ad = AbundanceData(cluster_sacc_data, 2) with pytest.raises(NotImplementedError): ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.SHEAR) + + +def test_observed_data_and_indices_no_data_throws(): + # pylint: disable=no-member + cc = sacc.standard_types.cluster_counts + + s = sacc.Sacc() + s.add_tracer("survey", "my_survey", 4000) + s.add_tracer("bin_z", "z_bin_tracer_1", 0, 2) + s.add_tracer("bin_z", "z_bin_tracer_2", 2, 4) + s.add_tracer("bin_richness", "mass_bin_tracer_1", 0, 2) + s.add_tracer("bin_richness", "mass_bin_tracer_2", 2, 4) + + s.add_data_point( + cc, + ("my_survey", "z_bin_tracer_1", "mass_bin_tracer_1"), + 1, + ) + s.add_data_point( + cc, + ("my_survey", "z_bin_tracer_1", "mass_bin_tracer_2"), + 1, + ) + + ad = AbundanceData(s, 2) + + with pytest.raises( + ValueError, match="The SACC file does not contain any tracers for the" + ): + ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.MASS) + + +def test_observed_data_and_indices_wrong_dimension_throws(cluster_sacc_data: sacc.Sacc): + ad = AbundanceData(cluster_sacc_data, 1) + + with pytest.raises(ValueError, match="The SACC file must contain"): + ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.MASS) From 1030be4931323b105c95daabcd53468fc21aeba8 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 4 Dec 2023 15:48:43 -0800 Subject: [PATCH 73/80] Changes to test marking. Updating verifying bin dimensions on the statistic. --- .../statistic/binned_cluster_number_counts.py | 6 +++ firecrown/models/cluster/kernel.py | 27 ----------- pytest.ini | 3 +- tests/test_cluster_abundance.py | 12 ++--- tests/test_cluster_kernels.py | 45 ++----------------- 5 files changed, 17 insertions(+), 76 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index bb2fbdaff..c9df35a2a 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -62,6 +62,12 @@ def read(self, sacc_data: sacc.Sacc) -> None: self.bin_edges = sacc_adapter.get_bin_edges( self.survey_name, self.cluster_properties ) + for bin_edge in self.bin_edges: + if bin_edge.dimension != self.bin_edges[0].dimension: + raise ValueError( + "The cluster number counts statistic requires all bins to be the " + "same dimension." + ) super().read(sacc_data) diff --git a/firecrown/models/cluster/kernel.py b/firecrown/models/cluster/kernel.py index 31c194273..3e094269c 100644 --- a/firecrown/models/cluster/kernel.py +++ b/firecrown/models/cluster/kernel.py @@ -165,30 +165,3 @@ def distribution( _z_proxy_limits: Tuple[float, float], ) -> npt.NDArray[np.float64]: return np.atleast_1d(1.0) - - -class DESY1PhotometricRedshift(Kernel): - """The DES Y1 photometric redshift uncertainties kernel. - - This kernel includes a spread in the photo-z estimates following the - observed spread in DES Y1.""" - - def __init__(self) -> None: - super().__init__(KernelType.Z_PROXY) - self.sigma_0 = 0.05 - - def distribution( - self, - _mass: npt.NDArray[np.float64], - z: npt.NDArray[np.float64], - _mass_proxy: npt.NDArray[np.float64], - z_proxy: npt.NDArray[np.float64], - _mass_proxy_limits: Tuple[float, float], - _z_proxy_limits: Tuple[float, float], - ) -> npt.NDArray[np.float64]: - sigma_z = self.sigma_0 * (1 + z) - prefactor = 1 / (np.sqrt(2.0 * np.pi) * sigma_z) - distribution = np.exp(-(1 / 2) * ((z_proxy - z) / sigma_z) ** 2.0) - numerator = prefactor * distribution - assert isinstance(numerator, np.ndarray) - return numerator diff --git a/pytest.ini b/pytest.ini index 5cec28a1a..f679a402e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,7 +6,8 @@ addopts = testpaths = tests markers = slow: Mark slow tests to ignore them unless they are requested - regression: Tests which cover the overall functionality of a feature. Typically carried out after significant changes. + integration: Tests to determine if independent units of code work together. Typically carried out after significant changes. + regression: Tests to verify the software still works as expected after changes. filterwarnings = ignore::DeprecationWarning:pkg_resources.*: ignore::DeprecationWarning:cobaya.*: diff --git a/tests/test_cluster_abundance.py b/tests/test_cluster_abundance.py index b5c1192ff..f811d8738 100644 --- a/tests/test_cluster_abundance.py +++ b/tests/test_cluster_abundance.py @@ -135,7 +135,7 @@ def test_abundance_comoving_returns_value(cluster_abundance: ClusterAbundance): assert np.all(result > 0) -@pytest.mark.regression +@pytest.mark.slow def test_abundance_massfunc_returns_value(cluster_abundance: ClusterAbundance): cosmo = pyccl.CosmologyVanillaLCDM() cluster_abundance.update_ingredients(cosmo) @@ -149,7 +149,7 @@ def test_abundance_massfunc_returns_value(cluster_abundance: ClusterAbundance): assert np.all(result > 0) -@pytest.mark.regression +@pytest.mark.slow def test_abundance_get_integrand( cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): @@ -164,7 +164,7 @@ def test_abundance_get_integrand( assert np.issubdtype(result.dtype, np.float64) -@pytest.mark.regression +@pytest.mark.slow def test_abundance_get_integrand_avg_mass( cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): @@ -179,7 +179,7 @@ def test_abundance_get_integrand_avg_mass( assert np.issubdtype(result.dtype, np.float64) -@pytest.mark.regression +@pytest.mark.slow def test_abundance_get_integrand_avg_redshift( cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): @@ -196,7 +196,7 @@ def test_abundance_get_integrand_avg_redshift( assert np.issubdtype(result.dtype, np.float64) -@pytest.mark.regression +@pytest.mark.slow def test_abundance_get_integrand_avg_mass_and_redshift( cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): @@ -212,7 +212,7 @@ def test_abundance_get_integrand_avg_mass_and_redshift( assert np.issubdtype(result.dtype, np.float64) -@pytest.mark.regression +@pytest.mark.slow def test_abundance_get_integrand_avg_not_implemented_throws( cluster_abundance: ClusterAbundance, integrable_kernel: Kernel, integrand_args ): diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index 6f30fbb50..ead52cc9a 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -6,21 +6,11 @@ Purity, KernelType, Kernel, - DESY1PhotometricRedshift, SpectroscopicRedshift, TrueMass, ) -def test_create_desy1_photometric_redshift_kernel(): - drk = DESY1PhotometricRedshift() - assert isinstance(drk, Kernel) - assert drk.kernel_type == KernelType.Z_PROXY - assert drk.is_dirac_delta is False - assert drk.integral_bounds is None - assert drk.has_analytic_sln is False - - def test_create_spectroscopic_redshift_kernel(): srk = SpectroscopicRedshift() assert isinstance(srk, Kernel) @@ -89,6 +79,7 @@ def test_true_mass_distribution(): ) +@pytest.mark.regression def test_purity_distribution(): pk = Purity() @@ -124,6 +115,7 @@ def test_purity_distribution(): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) +@pytest.mark.regression def test_purity_distribution_uses_mean(): pk = Purity() @@ -159,6 +151,7 @@ def test_purity_distribution_uses_mean(): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) +@pytest.mark.regression def test_completeness_distribution(): ck = Completeness() mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) @@ -190,35 +183,3 @@ def test_completeness_distribution(): assert isinstance(comp, np.ndarray) for ref, true in zip(comp, truth): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) - - -def test_des_photoz_kernel_distribution(): - dpk = DESY1PhotometricRedshift() - - mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - mass_proxy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - z = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - z_proxy = np.array([0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 1.01]) - - mass_proxy_limits = (1.0, 10.0) - z_proxy_limits = (0.11, 1.01) - - truth = [ - 7.134588921656481, - 6.557328601698999, - 6.065367634804254, - 5.641316284718016, - 5.272157878477569, - 4.9479710868093685, - 4.661070179674804, - 4.405413986167644, - 4.176191421334415, - 3.969525474770118, - ] - - spread = dpk.distribution( - mass, z, mass_proxy, z_proxy, mass_proxy_limits, z_proxy_limits - ) - assert isinstance(spread, np.ndarray) - for ref, true in zip(spread, truth): - assert ref == pytest.approx(true, rel=1e-7, abs=0.0) From 2b31acc8d1c91d207fd1afe0f6b868a178fdbf0f Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 4 Dec 2023 15:50:19 -0800 Subject: [PATCH 74/80] Removing numcosmo name from integrator tests. --- tests/test_cluster_integrators.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_cluster_integrators.py b/tests/test_cluster_integrators.py index 481aed993..acaf31a32 100644 --- a/tests/test_cluster_integrators.py +++ b/tests/test_cluster_integrators.py @@ -1,4 +1,4 @@ -"""Tests for the numcosmo integrator module.""" +"""Tests for the integrator module.""" from typing import Tuple from unittest.mock import Mock import numpy as np @@ -16,7 +16,7 @@ def fixture_integrator(request) -> Integrator: return request.param() -def test_numcosmo_set_integration_bounds_no_kernels( +def test_set_integration_bounds_no_kernels( empty_cluster_abundance: ClusterAbundance, integrator: Integrator ): z_array = np.linspace(0, 2, 10) @@ -40,7 +40,7 @@ def test_numcosmo_set_integration_bounds_no_kernels( } -def test_numcosmo_set_integration_bounds_dirac_delta( +def test_set_integration_bounds_dirac_delta( empty_cluster_abundance: ClusterAbundance, integrator: Integrator ): z_array = np.linspace(0, 2, 10) @@ -119,7 +119,7 @@ def test_numcosmo_set_integration_bounds_dirac_delta( } -def test_numcosmo_set_integration_bounds_integrable_kernels( +def test_set_integration_bounds_integrable_kernels( empty_cluster_abundance: ClusterAbundance, integrator: Integrator ): z_array = np.linspace(0, 2, 10) @@ -203,7 +203,7 @@ def test_numcosmo_set_integration_bounds_integrable_kernels( } -def test_numcosmo_set_integration_bounds_analytic_slns( +def test_set_integration_bounds_analytic_slns( empty_cluster_abundance: ClusterAbundance, integrator: Integrator ): z_array = np.linspace(0, 2, 10) @@ -258,7 +258,7 @@ def test_numcosmo_set_integration_bounds_analytic_slns( } -def test_numcosmo_integrator_integrate( +def test_integrator_integrate( empty_cluster_abundance: ClusterAbundance, integrator: Integrator ): empty_cluster_abundance.min_mass = 0 From 6cb8bffca342cae8a9f9abd69b2ea3dfdde3fed8 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 4 Dec 2023 16:43:44 -0800 Subject: [PATCH 75/80] Removing dimension from abundance data tests. Moving shared attributes to base class integrator Adding proper normalization to gaussian --- .../statistic/binned_cluster_number_counts.py | 3 +- firecrown/models/cluster/abundance_data.py | 11 ++---- .../models/cluster/integrator/integrator.py | 9 ++++- .../cluster/integrator/numcosmo_integrator.py | 7 ---- .../cluster/integrator/scipy_integrator.py | 5 --- firecrown/models/cluster/mass_proxy.py | 2 +- .../statistic/test_cluster_number_counts.py | 3 ++ tests/test_cluster_data.py | 34 +++++++------------ 8 files changed, 28 insertions(+), 46 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index c9df35a2a..761bcd605 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -43,14 +43,13 @@ def __init__( self.data_vector = DataVector.from_list([]) self.sky_area = 0.0 self.bin_edges: List[SaccBin] = [] - self.bin_dimensions = 2 def read(self, sacc_data: sacc.Sacc) -> None: # Build the data vector and indices needed for the likelihood if self.cluster_properties == ClusterProperty.NONE: raise ValueError("You must specify at least one cluster property.") - sacc_adapter = AbundanceData(sacc_data, self.bin_dimensions) + sacc_adapter = AbundanceData(sacc_data) self.sky_area = sacc_adapter.get_survey_tracer(self.survey_name).sky_area data, indices = sacc_adapter.get_observed_data_and_indices_by_survey( diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 7da865d4a..3bbc7c6d0 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -17,18 +17,12 @@ class AbundanceData: number count statistic. The data in this class is specific to a single survey name.""" - # Hard coded in SACC, how do we want to handle this? _survey_index = 0 _redshift_index = 1 _mass_index = 2 - def __init__( - self, - sacc_data: sacc.Sacc, - bin_dimensions: int, - ): + def __init__(self, sacc_data: sacc.Sacc): self.sacc_data = sacc_data - self.bin_dimensions = bin_dimensions def get_survey_tracer(self, survey_nm: str) -> SurveyTracer: """Returns the SurveyTracer for the specified survey name.""" @@ -96,8 +90,7 @@ def _all_bin_combinations_for_data_type(self, data_type: str) -> npt.NDArray: f"{data_type} data type." ) - # Add one because we don't include survey in the bin dimensions (for now) - if bins_combos_for_type.shape[1] != self.bin_dimensions + 1: + if bins_combos_for_type.shape[1] != 3: raise ValueError( "The SACC file must contain 3 tracers for the " "cluster_counts data type: cluster_survey, " diff --git a/firecrown/models/cluster/integrator/integrator.py b/firecrown/models/cluster/integrator/integrator.py index a28745156..4503753bd 100644 --- a/firecrown/models/cluster/integrator/integrator.py +++ b/firecrown/models/cluster/integrator/integrator.py @@ -5,7 +5,7 @@ """ import inspect from abc import ABC, abstractmethod -from typing import Tuple, Dict, get_args +from typing import Tuple, Dict, get_args, List from firecrown.models.cluster.abundance import ClusterAbundance, AbundanceIntegrand from firecrown.models.cluster.kernel import KernelType @@ -16,6 +16,13 @@ class Integrator(ABC): This class acts as an adapter around an integration library, and must provides a specific set of methods to be used to integrate a cluster abundance integral.""" + def __init__(self) -> None: + self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) + self.sky_area: float = 360**2 + self.integral_bounds: List[Tuple[float, float]] = [] + self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() + @abstractmethod def integrate( self, diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 3a07b7a08..8da4fe6fb 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -35,13 +35,6 @@ def __init__( self._absolute_tolerance = absolute_tolerance self._method = method or NumCosmoIntegralMethod.P_V - self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() - self.integral_bounds: List[Tuple[float, float]] = [] - - self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) - self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) - self.sky_area: float = 360**2 - def _integral_wrapper( self, integrand: AbundanceIntegrand, diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index 003858491..f1b3b3800 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -21,13 +21,8 @@ def __init__( self._relative_tolerance = relative_tolerance self._absolute_tolerance = absolute_tolerance - self.integral_bounds: List[Tuple[float, float]] = [] self.integral_args_lkp: Dict[KernelType, int] = self._default_integral_args() - self.z_proxy_limits: Tuple[float, float] = (-1.0, -1.0) - self.mass_proxy_limits: Tuple[float, float] = (-1.0, -1.0) - self.sky_area: float = 360**2 - def _integral_wrapper( self, integrand: AbundanceIntegrand, diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index cd35689be..cc91d43d3 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -96,7 +96,7 @@ def _distribution_unbinned( proxy_sigma = self.get_proxy_sigma(mass, z) result = np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( - 2 * np.pi * proxy_sigma + np.sqrt(2 * np.pi) * proxy_sigma ) assert isinstance(result, np.ndarray) return result diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index ff932e60c..494daaf5a 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -63,6 +63,7 @@ def test_read(cluster_sacc_data: sacc.Sacc): assert bnc.sky_area == 4000 assert len(bnc.bin_edges) == 2 assert len(bnc.data_vector) == 2 + assert bnc.sacc_indices is not None assert len(bnc.sacc_indices) == 2 bnc = BinnedClusterNumberCounts(ClusterProperty.MASS, "my_survey", integrator) @@ -70,6 +71,7 @@ def test_read(cluster_sacc_data: sacc.Sacc): assert bnc.sky_area == 4000 assert len(bnc.bin_edges) == 2 assert len(bnc.data_vector) == 2 + assert bnc.sacc_indices is not None assert len(bnc.sacc_indices) == 2 bnc = BinnedClusterNumberCounts( @@ -79,6 +81,7 @@ def test_read(cluster_sacc_data: sacc.Sacc): assert bnc.sky_area == 4000 assert len(bnc.bin_edges) == 2 assert len(bnc.data_vector) == 4 + assert bnc.sacc_indices is not None assert len(bnc.sacc_indices) == 4 diff --git a/tests/test_cluster_data.py b/tests/test_cluster_data.py index 5556dd92a..4c6084902 100644 --- a/tests/test_cluster_data.py +++ b/tests/test_cluster_data.py @@ -7,9 +7,8 @@ def test_create_abundance_data(): s = sacc.Sacc() - ad = AbundanceData(s, 2) + ad = AbundanceData(s) - assert ad.bin_dimensions == 2 # pylint: disable=protected-access assert ad._mass_index == 2 # pylint: disable=protected-access @@ -19,7 +18,7 @@ def test_create_abundance_data(): def test_get_survey_tracer_missing_survey_name(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) with pytest.raises( ValueError, match="The SACC file does not contain the SurveyTracer the_black_lodge.", @@ -28,7 +27,7 @@ def test_get_survey_tracer_missing_survey_name(cluster_sacc_data: sacc.Sacc): def test_get_survey_tracer_wrong_tracer_type(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) with pytest.raises( ValueError, match="The SACC tracer z_bin_tracer_1 is not a SurveyTracer.", @@ -37,26 +36,26 @@ def test_get_survey_tracer_wrong_tracer_type(cluster_sacc_data: sacc.Sacc): def test_get_survey_tracer_returns_survey_tracer(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) survey = ad.get_survey_tracer("my_survey") assert survey is not None assert isinstance(survey, sacc.tracers.SurveyTracer) def test_get_bin_edges_cluster_counts(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) bins = ad.get_bin_edges("my_survey", ClusterProperty.COUNTS) assert len(bins) == 2 def test_get_bin_edges_cluster_mass(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) bins = ad.get_bin_edges("my_survey", ClusterProperty.MASS) assert len(bins) == 2 def test_get_bin_edges_counts_and_mass(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) bins = ad.get_bin_edges( "my_survey", (ClusterProperty.MASS | ClusterProperty.COUNTS) ) @@ -66,7 +65,7 @@ def test_get_bin_edges_counts_and_mass(cluster_sacc_data: sacc.Sacc): def test_get_bin_edges_not_implemented_cluster_property_throws( cluster_sacc_data: sacc.Sacc, ): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) with pytest.raises(NotImplementedError): ad.get_bin_edges("my_survey", ClusterProperty.SHEAR) @@ -74,7 +73,7 @@ def test_get_bin_edges_not_implemented_cluster_property_throws( def test_observed_data_and_indices_by_survey_cluster_counts( cluster_sacc_data: sacc.Sacc, ): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) data, indices = ad.get_observed_data_and_indices_by_survey( "my_survey", ClusterProperty.COUNTS ) @@ -85,7 +84,7 @@ def test_observed_data_and_indices_by_survey_cluster_counts( def test_observed_data_and_indices_by_survey_cluster_mass( cluster_sacc_data: sacc.Sacc, ): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) data, indices = ad.get_observed_data_and_indices_by_survey( "my_survey", ClusterProperty.MASS ) @@ -96,7 +95,7 @@ def test_observed_data_and_indices_by_survey_cluster_mass( def test_observed_data_and_indices_by_survey_cluster_counts_and_mass( cluster_sacc_data: sacc.Sacc, ): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) data, indices = ad.get_observed_data_and_indices_by_survey( "my_survey", ClusterProperty.MASS | ClusterProperty.COUNTS ) @@ -107,7 +106,7 @@ def test_observed_data_and_indices_by_survey_cluster_counts_and_mass( def test_observed_data_and_indices_by_survey_not_implemented_throws( cluster_sacc_data: sacc.Sacc, ): - ad = AbundanceData(cluster_sacc_data, 2) + ad = AbundanceData(cluster_sacc_data) with pytest.raises(NotImplementedError): ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.SHEAR) @@ -134,16 +133,9 @@ def test_observed_data_and_indices_no_data_throws(): 1, ) - ad = AbundanceData(s, 2) + ad = AbundanceData(s) with pytest.raises( ValueError, match="The SACC file does not contain any tracers for the" ): ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.MASS) - - -def test_observed_data_and_indices_wrong_dimension_throws(cluster_sacc_data: sacc.Sacc): - ad = AbundanceData(cluster_sacc_data, 1) - - with pytest.raises(ValueError, match="The SACC file must contain"): - ad.get_observed_data_and_indices_by_survey("my_survey", ClusterProperty.MASS) From 7a87d141ce6422d7fadf254621cb16ed47d59564 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Tue, 5 Dec 2023 16:49:23 -0800 Subject: [PATCH 76/80] Forgot to fix imports. --- firecrown/models/cluster/integrator/numcosmo_integrator.py | 2 +- firecrown/models/cluster/integrator/scipy_integrator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/firecrown/models/cluster/integrator/numcosmo_integrator.py b/firecrown/models/cluster/integrator/numcosmo_integrator.py index 8da4fe6fb..d3105307d 100644 --- a/firecrown/models/cluster/integrator/numcosmo_integrator.py +++ b/firecrown/models/cluster/integrator/numcosmo_integrator.py @@ -2,7 +2,7 @@ This module holds the NumCosmo implementation of the integrator classes """ -from typing import Tuple, Callable, Dict, Sequence, List, Optional +from typing import Tuple, Callable, Sequence, Optional from enum import Enum import numpy as np import numpy.typing as npt diff --git a/firecrown/models/cluster/integrator/scipy_integrator.py b/firecrown/models/cluster/integrator/scipy_integrator.py index f1b3b3800..87bc8420e 100644 --- a/firecrown/models/cluster/integrator/scipy_integrator.py +++ b/firecrown/models/cluster/integrator/scipy_integrator.py @@ -2,7 +2,7 @@ This module holds the scipy implementation of the integrator classes """ -from typing import Callable, Dict, Tuple, List +from typing import Callable, Dict, Tuple import numpy as np import numpy.typing as npt from scipy.integrate import nquad From f86092034bdc9bb67bb57cb8f413558103ad3c87 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 6 Dec 2023 07:09:36 -0800 Subject: [PATCH 77/80] Making pylint happy with parenthesis --- .../gauss_family/statistic/binned_cluster_number_counts.py | 3 ++- firecrown/models/cluster/abundance.py | 3 ++- firecrown/models/cluster/abundance_data.py | 3 ++- firecrown/models/cluster/binning.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 761bcd605..3913d6283 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -83,7 +83,8 @@ def compute_theory_vector(self, tools: ModelingTools) -> TheoryVector: cluster_counts = self.get_binned_cluster_counts(tools) for cl_property in ClusterProperty: - if not (cl_property & self.cluster_properties): + include_prop = cl_property & self.cluster_properties + if not include_prop: continue if cl_property == ClusterProperty.COUNTS: diff --git a/firecrown/models/cluster/abundance.py b/firecrown/models/cluster/abundance.py index 9d74677cb..6d836ff1c 100644 --- a/firecrown/models/cluster/abundance.py +++ b/firecrown/models/cluster/abundance.py @@ -153,7 +153,8 @@ def integrand( return integrand for cluster_prop in ClusterProperty: - if not (cluster_prop & average_properties): + include_prop = cluster_prop & average_properties + if not include_prop: continue if cluster_prop == ClusterProperty.MASS: integrand *= mass diff --git a/firecrown/models/cluster/abundance_data.py b/firecrown/models/cluster/abundance_data.py index 3bbc7c6d0..a37975952 100644 --- a/firecrown/models/cluster/abundance_data.py +++ b/firecrown/models/cluster/abundance_data.py @@ -53,7 +53,8 @@ def get_observed_data_and_indices_by_survey( sacc_indices = [] for cluster_property in ClusterProperty: - if not (cluster_property & properties): + include_prop = cluster_property & properties + if not include_prop: continue if cluster_property == ClusterProperty.COUNTS: diff --git a/firecrown/models/cluster/binning.py b/firecrown/models/cluster/binning.py index c761c618e..5e75312bc 100644 --- a/firecrown/models/cluster/binning.py +++ b/firecrown/models/cluster/binning.py @@ -61,7 +61,7 @@ def __eq__(self, other: object) -> bool: if self.dimension != other.dimension: return False - for i, my_bin in enumerate(self.coordinate_bins): + for my_bin in self.coordinate_bins: other_bin = [ x for x in other.coordinate_bins if x.tracer_type == my_bin.tracer_type ] From 0b7d5cf08cb2d0bbc6676228b69e2910972fd571 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Wed, 6 Dec 2023 07:52:12 -0800 Subject: [PATCH 78/80] Fixing pylint CI errors. --- tests/test_cluster_binning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_cluster_binning.py b/tests/test_cluster_binning.py index 3e8ab1699..2885272be 100644 --- a/tests/test_cluster_binning.py +++ b/tests/test_cluster_binning.py @@ -1,5 +1,6 @@ -from firecrown.models.cluster.binning import SaccBin +"""Tests for the cluster binning module""" import sacc +from firecrown.models.cluster.binning import SaccBin def test_create_sacc_bin_with_correct_dimension(): From bd1d08f9dce9192fc6f7ed5624c036871c91b063 Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Thu, 7 Dec 2023 14:31:39 -0800 Subject: [PATCH 79/80] Adding a unit test to check normalization of MurataUnbinned Some PR review changes Adding more tests for the binning module. --- .../statistic/binned_cluster_number_counts.py | 12 +-- firecrown/models/cluster/binning.py | 10 ++- firecrown/models/cluster/mass_proxy.py | 6 +- pytest.ini | 2 +- .../statistic/test_cluster_number_counts.py | 6 +- tests/test_cluster_binning.py | 87 ++++++++++++++++++- tests/test_cluster_kernels.py | 6 +- tests/test_cluster_mass_richness.py | 31 +++++++ 8 files changed, 142 insertions(+), 18 deletions(-) diff --git a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py index 3913d6283..8d5e9e01f 100644 --- a/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py +++ b/firecrown/likelihood/gauss_family/statistic/binned_cluster_number_counts.py @@ -42,7 +42,7 @@ def __init__( self.integrator = integrator self.data_vector = DataVector.from_list([]) self.sky_area = 0.0 - self.bin_edges: List[SaccBin] = [] + self.bins: List[SaccBin] = [] def read(self, sacc_data: sacc.Sacc) -> None: # Build the data vector and indices needed for the likelihood @@ -58,11 +58,11 @@ def read(self, sacc_data: sacc.Sacc) -> None: self.data_vector = DataVector.from_list(data) self.sacc_indices = np.array(indices) - self.bin_edges = sacc_adapter.get_bin_edges( + self.bins = sacc_adapter.get_bin_edges( self.survey_name, self.cluster_properties ) - for bin_edge in self.bin_edges: - if bin_edge.dimension != self.bin_edges[0].dimension: + for bin_edge in self.bins: + if bin_edge.dimension != self.bins[0].dimension: raise ValueError( "The cluster number counts statistic requires all bins to be the " "same dimension." @@ -111,7 +111,7 @@ def get_binned_cluster_property( assert tools.cluster_abundance is not None cluster_masses = [] - for bin_edge, counts in zip(self.bin_edges, cluster_counts): + for bin_edge, counts in zip(self.bins, cluster_counts): integrand = tools.cluster_abundance.get_integrand( average_properties=cluster_properties ) @@ -137,7 +137,7 @@ def get_binned_cluster_counts(self, tools: ModelingTools) -> List[float]: assert tools.cluster_abundance is not None cluster_counts = [] - for bin_edge in self.bin_edges: + for bin_edge in self.bins: self.integrator.set_integration_bounds( tools.cluster_abundance, self.sky_area, diff --git a/firecrown/models/cluster/binning.py b/firecrown/models/cluster/binning.py index 5e75312bc..f4183b718 100644 --- a/firecrown/models/cluster/binning.py +++ b/firecrown/models/cluster/binning.py @@ -47,11 +47,17 @@ def __init__(self, bins: List[sacc.BaseTracer]): @property def z_edges(self) -> Tuple[float, float]: - return self.coordinate_bins[0].lower, self.coordinate_bins[0].upper + z_bin = [x for x in self.coordinate_bins if x.tracer_type == "bin_z"] + if len(z_bin) != 1: + raise ValueError("SaccBin must have exactly one z bin") + return z_bin[0].lower, z_bin[0].upper @property def mass_proxy_edges(self) -> Tuple[float, float]: - return self.coordinate_bins[1].lower, self.coordinate_bins[1].upper + mass_bin = [x for x in self.coordinate_bins if x.tracer_type == "bin_richness"] + if len(mass_bin) != 1: + raise ValueError("SaccBin must have exactly one richness bin") + return mass_bin[0].lower, mass_bin[0].upper def __eq__(self, other: object) -> bool: """Two bins are equal if they have the same lower/upper bound.""" diff --git a/firecrown/models/cluster/mass_proxy.py b/firecrown/models/cluster/mass_proxy.py index cc91d43d3..f64a4f6a9 100644 --- a/firecrown/models/cluster/mass_proxy.py +++ b/firecrown/models/cluster/mass_proxy.py @@ -95,9 +95,11 @@ def _distribution_unbinned( proxy_mean = self.get_proxy_mean(mass, z) proxy_sigma = self.get_proxy_sigma(mass, z) - result = np.exp(-0.5 * (mass_proxy - proxy_mean) ** 2 / proxy_sigma**2) / ( - np.sqrt(2 * np.pi) * proxy_sigma + normalization = 1 / np.sqrt(2 * np.pi * proxy_sigma**2) + result = normalization * np.exp( + -0.5 * ((mass_proxy - proxy_mean) / proxy_sigma) ** 2 ) + assert isinstance(result, np.ndarray) return result diff --git a/pytest.ini b/pytest.ini index f679a402e..758c873a0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,7 @@ testpaths = tests markers = slow: Mark slow tests to ignore them unless they are requested integration: Tests to determine if independent units of code work together. Typically carried out after significant changes. - regression: Tests to verify the software still works as expected after changes. + precision_sensitive: Tests that are sensitive to the numerical precision used in calculations. filterwarnings = ignore::DeprecationWarning:pkg_resources.*: ignore::DeprecationWarning:cobaya.*: diff --git a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py index 494daaf5a..ff34e4d88 100644 --- a/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py +++ b/tests/likelihood/gauss_family/statistic/test_cluster_number_counts.py @@ -61,7 +61,7 @@ def test_read(cluster_sacc_data: sacc.Sacc): bnc = BinnedClusterNumberCounts(ClusterProperty.COUNTS, "my_survey", integrator) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 - assert len(bnc.bin_edges) == 2 + assert len(bnc.bins) == 2 assert len(bnc.data_vector) == 2 assert bnc.sacc_indices is not None assert len(bnc.sacc_indices) == 2 @@ -69,7 +69,7 @@ def test_read(cluster_sacc_data: sacc.Sacc): bnc = BinnedClusterNumberCounts(ClusterProperty.MASS, "my_survey", integrator) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 - assert len(bnc.bin_edges) == 2 + assert len(bnc.bins) == 2 assert len(bnc.data_vector) == 2 assert bnc.sacc_indices is not None assert len(bnc.sacc_indices) == 2 @@ -79,7 +79,7 @@ def test_read(cluster_sacc_data: sacc.Sacc): ) bnc.read(cluster_sacc_data) assert bnc.sky_area == 4000 - assert len(bnc.bin_edges) == 2 + assert len(bnc.bins) == 2 assert len(bnc.data_vector) == 4 assert bnc.sacc_indices is not None assert len(bnc.sacc_indices) == 4 diff --git a/tests/test_cluster_binning.py b/tests/test_cluster_binning.py index 2885272be..04abbdc35 100644 --- a/tests/test_cluster_binning.py +++ b/tests/test_cluster_binning.py @@ -1,6 +1,7 @@ """Tests for the cluster binning module""" import sacc -from firecrown.models.cluster.binning import SaccBin +from firecrown.models.cluster.binning import SaccBin, NDimensionalBin +from unittest.mock import Mock def test_create_sacc_bin_with_correct_dimension(): @@ -10,3 +11,87 @@ def test_create_sacc_bin_with_correct_dimension(): sb = SaccBin(tracers) assert sb is not None assert sb.dimension == i + + +def test_sacc_bin_z_edges(): + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb = SaccBin([tracer_z, tracer_lambda]) + assert sb.z_edges == (0, 1) + + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb = SaccBin([tracer_lambda, tracer_z]) + assert sb.z_edges == (0, 1) + + +def test_sacc_bin_richness_edges(): + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb = SaccBin([tracer_z, tracer_lambda]) + assert sb.mass_proxy_edges == (4, 5) + + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb = SaccBin([tracer_lambda, tracer_z]) + assert sb.mass_proxy_edges == (4, 5) + + +def test_equal_sacc_bins_are_equal(): + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb1 = SaccBin([tracer_z, tracer_lambda]) + sb2 = SaccBin([tracer_z, tracer_lambda]) + + assert sb1 == sb2 + assert sb1 is not sb2 + + sb1 = SaccBin([tracer_lambda, tracer_z]) + sb2 = SaccBin([tracer_z, tracer_lambda]) + + assert sb1 == sb2 + assert sb1 is not sb2 + + +def test_sacc_bin_different_edges_not_equal(): + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb1 = SaccBin([tracer_z, tracer_lambda]) + + tracer_z = sacc.tracers.BinZTracer("", 0, 2) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb2 = SaccBin([tracer_z, tracer_lambda]) + + assert sb1 != sb2 + + tracer_z = sacc.tracers.BinZTracer("", -1, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb2 = SaccBin([tracer_z, tracer_lambda]) + + assert sb1 != sb2 + + tracer_z = sacc.tracers.BinZTracer("", -1, 2) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb2 = SaccBin([tracer_z, tracer_lambda]) + + assert sb1 != sb2 + + +def test_sacc_bin_different_dimensions_not_equal(): + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_z2 = sacc.tracers.BinZTracer("", 1, 2) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb1 = SaccBin([tracer_z, tracer_z2, tracer_lambda]) + sb2 = SaccBin([tracer_z, tracer_lambda]) + + assert sb1 != sb2 + + +def test_sacc_bin_must_be_equal_type(): + other_bin = Mock(spec=NDimensionalBin) + + tracer_z = sacc.tracers.BinZTracer("", 0, 1) + tracer_lambda = sacc.tracers.BinRichnessTracer("", 4, 5) + sb = SaccBin([tracer_z, tracer_lambda]) + + assert sb != other_bin diff --git a/tests/test_cluster_kernels.py b/tests/test_cluster_kernels.py index ead52cc9a..1719bdbf7 100644 --- a/tests/test_cluster_kernels.py +++ b/tests/test_cluster_kernels.py @@ -79,7 +79,7 @@ def test_true_mass_distribution(): ) -@pytest.mark.regression +@pytest.mark.precision_sensitive def test_purity_distribution(): pk = Purity() @@ -115,7 +115,7 @@ def test_purity_distribution(): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) -@pytest.mark.regression +@pytest.mark.precision_sensitive def test_purity_distribution_uses_mean(): pk = Purity() @@ -151,7 +151,7 @@ def test_purity_distribution_uses_mean(): assert ref == pytest.approx(true, rel=1e-7, abs=0.0) -@pytest.mark.regression +@pytest.mark.precision_sensitive def test_completeness_distribution(): ck = Completeness() mass = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) diff --git a/tests/test_cluster_mass_richness.py b/tests/test_cluster_mass_richness.py index c8e7fc532..7914cdcc9 100644 --- a/tests/test_cluster_mass_richness.py +++ b/tests/test_cluster_mass_richness.py @@ -1,6 +1,7 @@ """Tests for the cluster mass richness module.""" import pytest import numpy as np +from scipy.integrate import quad from firecrown.models.cluster.mass_proxy import ( MurataBinned, MurataUnbinned, @@ -200,3 +201,33 @@ def test_cluster_murata_unbinned_distribution(murata_unbinned_relation: MurataUn assert probability_1 <= probability_0 else: assert probability_1 >= probability_0 + + +@pytest.mark.precision_sensitive +def test_cluster_murata_unbinned_distribution_is_normalized( + murata_unbinned_relation: MurataUnbinned, +): + for mass_i, z_i in zip(np.linspace(7.0, 26.0, 20), np.geomspace(1.0e-18, 2.0, 20)): + mass = np.atleast_1d(mass_i) + z = np.atleast_1d(z_i) + + mean = murata_unbinned_relation.get_proxy_mean(mass, z)[0] + sigma = murata_unbinned_relation.get_proxy_sigma(mass, z)[0] + mass_proxy_limits = (mean - 5 * sigma, mean + 5 * sigma) + + def integrand(mass_proxy): + """Evaluate the unbinned distribution at fixed mass and redshift.""" + # pylint: disable=cell-var-from-loop + return murata_unbinned_relation.distribution( + mass, z, mass_proxy, z, mass_proxy_limits, (0.0, 1.0) + ) + + result, _ = quad( + integrand, + mass_proxy_limits[0], + mass_proxy_limits[1], + epsabs=1e-12, + epsrel=1e-12, + ) + + assert result == pytest.approx(1.0, rel=1.0e-6, abs=0.0) From 5009b56c67f17a57130890024f348daaca0e67fd Mon Sep 17 00:00:00 2001 From: Matt Kwiecien Date: Mon, 11 Dec 2023 08:19:59 -0800 Subject: [PATCH 80/80] Pylint import order error. --- tests/test_cluster_binning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cluster_binning.py b/tests/test_cluster_binning.py index 04abbdc35..2304e1a65 100644 --- a/tests/test_cluster_binning.py +++ b/tests/test_cluster_binning.py @@ -1,7 +1,7 @@ """Tests for the cluster binning module""" +from unittest.mock import Mock import sacc from firecrown.models.cluster.binning import SaccBin, NDimensionalBin -from unittest.mock import Mock def test_create_sacc_bin_with_correct_dimension():