From e466c03490a8f83291ea437d4ad25c0b0b2c875f Mon Sep 17 00:00:00 2001 From: Mark Jones Date: Fri, 23 Jul 2021 16:15:34 -0400 Subject: [PATCH] Release PyPWA 3.4 This includes some minor fixes, depreciates some features, and includes MCMC. --- CHANGELOG.md | 11 + PyPWA/progs/masking.py | 8 +- PyPWA/progs/simulation.py | 319 ----------------------- setup.py | 9 +- tests/{ => system_tests}/test_2dgauss.py | 0 tests/system_tests/test_binning_prog.py | 0 tests/system_tests/test_masking_prog.py | 39 +++ tests/test_full_program_execution.py | 43 --- 8 files changed, 58 insertions(+), 371 deletions(-) delete mode 100644 PyPWA/progs/simulation.py rename tests/{ => system_tests}/test_2dgauss.py (100%) create mode 100644 tests/system_tests/test_binning_prog.py create mode 100644 tests/system_tests/test_masking_prog.py delete mode 100644 tests/test_full_program_execution.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b513b158..667b184e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All changes important to the user will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/) +## [3.4.0] - 2021-7-23 +### Added +- Peter's emcee wrapper, available at PyPWA.mcmc +### Changed +- System tests are now located in tests/system_tests +- PyMask will now return exit values on call +### Removed +- PySimulate has been removed since it was limited in use, and it's + functionality has been consumed by the PyPWA scripting libs. + + ## [3.3.0] - 2021-6-20 ### Added - 2D Gauss introductory tutorial to the documentation diff --git a/PyPWA/progs/masking.py b/PyPWA/progs/masking.py index 10222ecb..35c41ee9 100644 --- a/PyPWA/progs/masking.py +++ b/PyPWA/progs/masking.py @@ -45,7 +45,7 @@ def start_masking(arguments: List[str] = sys.argv[1:]): output_file = file_manager.get_writer(args.output, input_file.data_type) else: print(f"{args.input} must exist!") - sys.exit(1) + return 1 # Load the correct file if args.use_or: @@ -54,7 +54,7 @@ def start_masking(arguments: List[str] = sys.argv[1:]): logic = npy.logical_xor elif args.use_or and args.use_xor: print("Only select OR or XOR, not both!") - sys.exit(1) + return 1 else: logic = npy.logical_and @@ -71,7 +71,7 @@ def start_masking(arguments: List[str] = sys.argv[1:]): current_pf = file_manager.parse(mask_file) else: print(f"{mask_file} must exist!") - sys.exit(1) + return 1 # Convert selection array to a boolean mask. if "u8" == current_pf.dtype or "u4" == current_pf.dtype: @@ -93,7 +93,7 @@ def start_masking(arguments: List[str] = sys.argv[1:]): f"Masking data isn't the same length as input!" f" Mask is {len(pf)} and input is {len(input_file)}." ) - sys.exit(1) + return 1 # Setup description if pf.all() == 1: diff --git a/PyPWA/progs/simulation.py b/PyPWA/progs/simulation.py deleted file mode 100644 index ae7a74e1..00000000 --- a/PyPWA/progs/simulation.py +++ /dev/null @@ -1,319 +0,0 @@ -# coding=utf-8 -# -# PyPWA, a scientific analysis toolkit. -# Copyright (C) 2016 JLab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -Entry Point for Simulate -""" - -import argparse -import copy -import logging -import multiprocessing -import sys -from pathlib import Path -from typing import Any, Dict, List, Tuple - -import numpy as npy -import yaml - -from PyPWA import info as _info -from PyPWA.libs import configuration -from PyPWA.libs import function -from PyPWA.libs import simulate -from PyPWA.libs.file import project, processor - -__credits__ = ["Mark Jones"] -__author__ = _info.AUTHOR -__version__ = _info.VERSION - - -_LOGGER = logging.getLogger(__file__) - -# Function, Intensity, Setup, -# Parameters, -# Data, Slot Name, Output, Enable Cache -_SETTINGS = Tuple[ - Path, str, str, - Dict[str, npy.float64], - Path, str, Path, bool -] - -_EXAMPLE = { - "Version": 1, - "Processes": multiprocessing.cpu_count(), - "Function": { - "Path": 'function.py', - "Amplitude Name": "MyAmplitude" - }, - - "Parameters": { - "A1": 1, - "A2": 2, - "A3": 3.1 - }, - - "Data": { - "Path": "data.type", - "Output": "output.txt" - } -} - -_TEMPLATE = { - "Version": int, - "Processes": int, - "Function": { - "Path": str, - "Amplitude Name": str, - "Intensity Name": str, - "Setup Name": str - }, - "Parameters": dict(), - "Data": { - "Path": str, - "Slot": str, - "Output": str - } -} - - -def simulation(arguments: List[str] = sys.argv[1:]): - args = _arguments(arguments) - - config = dict() - if args.type == "config": - if isinstance(args.config, Path) or args.example: - config = _handle_configuration(args) # Will exit if printing config - else: - print("Try 'pysimulate config --help") - sys.exit(1) - - print("Parsing provided parameters") - function_path, intensity_name, setup_name, \ - parameters, data_path, folder_name, output, use_cache = \ - _process_arguments(config, args) - - print("Loading data") - if folder_name: - factory = project.ProjectDatabase(data_path, "r") - data = factory.get_folder(folder_name) - - else: - data = processor.DataProcessor(use_cache).parse(data_path) - - print("Loading functions") - try: - intensity = function.load(function_path, intensity_name) - except ImportError as error: - print("Failed to load the intensity function!") - raise error - - try: - setup = function.load(function_path, setup_name) - except ImportError as error: - print("Failed to load the setup function!") - raise error - - print("Starting Simulation") - rejection = simulate.monte_carlo_simulation( - intensity, setup, parameters, data - ) - - if folder_name: - data.data.add_data(data.data.PASSFAIL, rejection) - - if output: - processor.DataProcessor().write(output, rejection) - - if not folder_name and not output: - _LOGGER.warn("No output was provided! Using stdout for output!") - [print(int(x)) for x in rejection] - - -def _arguments(args: List[str]) -> argparse.ArgumentParser.parse_args: - arguments = argparse.ArgumentParser() - - subparsers = arguments.add_subparsers(dest="type") - - config_subparser = subparsers.add_parser("config") - - config_subparser.add_argument( - "config", type=Path, nargs="?", - help="Use settings from a pre-made configuration file" - ) - - config_subparser.add_argument( - "--example", action="store_true", - help="Print an example configuration for PySimulate" - ) - - arguments.add_argument( - "--data", type=Path, metavar="DATA_FILE", - help="Data File or table to use for simulation" - ) - - arguments.add_argument( - "--output", type=Path, metavar="OUTPUT_FILE", - help="File to right rejection list out too, " - "omit if you are using tables" - ) - - arguments.add_argument( - "--folder", metavar="FOLDER_NAME", - help="The name of the slot to load from in the table. Omitted if not" - " using a table." - ) - - arguments.add_argument( - "--disable_cache", action="store_true", - help="Disable caching when loading data from a file." - ) - - arguments.add_argument( - "--param", "-p", nargs=2, action="append", metavar=("NAME", "VALUE"), - help="Parameters to simulate with, as: parameter_name value" - ) - - arguments.add_argument( - "--function", metavar="PYTHON_FILE", type=Path, - help="Python source file containing the functions for intensity for " - "the simulation" - ) - - arguments.add_argument( - "--amplitude", metavar="AMPLITUDE_NAME", default="MyAmplitude", - help="Name of the intensity function inside the source file." - ) - - return arguments.parse_args(args) - - -def _handle_configuration(args: argparse.Namespace) -> Dict[str, Any]: - - if args.example: - if args.config: - configuration.write(args.config, _EXAMPLE) - else: - print(yaml.dump(_EXAMPLE)) - sys.exit() - - if args.config.exists(): - return configuration.parse(args.config, _TEMPLATE) - else: - raise ValueError(f"{args.conf.name} doesn't exist!") - - -def _process_arguments( - config: Dict[str, Any], args: argparse.Namespace) -> _SETTINGS: - """Extracts information from both arguments and configuration file - - - Layout - ------ - - Fill out Dictionary - - Convert strings to paths - - Replace values with provided arguments - - Set extra variables - - Handle KeyErrors with specific values that don't have defaults - """ - combined = copy.deepcopy(config) - - # Fill in root keys in dictionary if not provided - if "Data" not in combined: - combined["Data"] = dict() - - if "Function" not in combined: - combined["Function"] = dict() - - if "Parameters" not in combined: - combined["Parameters"] = dict() - else: - for key in combined["Parameters"]: - combined["Parameters"][key] = npy.float(combined["Parameters"][key]) - - # Convert str to Path, if in configuration file - if "Path" in combined["Data"]: - combined["Data"]["Path"] = Path(combined["Data"]["Path"]) - combined["Data"]["Path"] = combined["Data"]["Path"].resolve() - - if "Path" in combined["Function"]: - combined["Function"]["Path"] = Path(combined["Function"]["Path"]) - - if "Output" in combined["Data"]: - combined["Data"]["Output"] = Path(combined["Data"]["Output"]) - - # Replace data path if path is provided and exists - if args.data and args.data.exists(): - combined["Data"]["Path"] = args.data - - # Even if we don't have a slot, the value needs to be set - if args.folder or "Folder" not in combined["Data"]: - combined["Data"]["Folder"] = args.folder - - # Replace function path if provided and exists - if args.function and args.function.exists(): - combined["Function"]["Path"] = args.function - - # Replace output if provided it or set it if unset - if args.output or "Output" not in combined["Data"]: - combined["Data"]["Output"] = args.output - - # Replace intensity name if provided it or set it if unset - if "Intensity Name" not in combined["Function"] or ( - "Intensity Name" in combined["Function"] and - args.intensity != "intensity"): - combined["Function"]["Intensity Name"] = args.intensity - - # Replace setup name if provided it or set it if unset - if "Setup Name" not in combined["Function"] or ( - "Setup Name" in combined["Function"] and - args.setup != "setup"): - combined["Function"]["Setup Name"] = args.setup - - # Append provided parameters - if args.param: - for param in args.param: - combined["Parameters"][param[0]] = npy.float(param[1]) - - # Set caching variable - combined["Data"]["Cache"] = args.disable_cache - - # We only check the values that don't have defaults and have to be set - try: - func_path = combined["Function"]["Path"] - except KeyError: - print( - "You must provide a function path either directly or in the" - " configuration file" - ) - sys.exit(126) - - try: - data_path = combined["Data"]["Path"] - except KeyError: - print( - "You must provide a path to the data either directly or in the" - " configuration file" - ) - sys.exit(126) - - return ( - func_path, combined["Function"]["Intensity Name"], - combined["Function"]["Setup Name"], combined["Parameters"], - data_path, combined["Data"]["Folder"], combined["Data"]["Output"], - combined["Data"]["Cache"] - ) diff --git a/setup.py b/setup.py index 63e717f0..6a27d2fb 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ __author__ = "PyPWA Team and Contributors" __license__ = "GPLv3" -__version__ = "3.3.0" +__version__ = "3.4.0" __email__ = "pypwa@jlab.org" __status__ = "development" @@ -30,9 +30,7 @@ entry_points = { "console_scripts": [ f"pymask = {progs}.masking:start_masking", - f"pybin = {progs}.binner:start_binning", -# f"pysimulate = {progs}.simulation:simulation", -# f"pyfit = {progs}.pyfit:start_fitting" + f"pybin = {progs}.binner:start_binning" ] } @@ -48,7 +46,8 @@ "pandas", # A powerful statistics package that's used everywhere "openpyxl", # Provides support for XLXS, used for resonance, "matplotlib", # Adds support for plotting - "numexpr" # Accelerates numpy by removing intermediate steps + "numexpr", # Accelerates numpy by removing intermediate steps + "emcee", # Provides MCMC fitting ] extras = { diff --git a/tests/test_2dgauss.py b/tests/system_tests/test_2dgauss.py similarity index 100% rename from tests/test_2dgauss.py rename to tests/system_tests/test_2dgauss.py diff --git a/tests/system_tests/test_binning_prog.py b/tests/system_tests/test_binning_prog.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/system_tests/test_masking_prog.py b/tests/system_tests/test_masking_prog.py new file mode 100644 index 00000000..17891441 --- /dev/null +++ b/tests/system_tests/test_masking_prog.py @@ -0,0 +1,39 @@ +from PyPWA.progs import masking +from pathlib import Path + + +def test_masking_errors_with_too_many_flags(): + result = masking.start_masking( + [ + "--use_or", "--use_xor", + "-i", "tests/test_data/docs/large.gamp", + "-m", "tests/test_data/docs/set2.pf", + "-o", "error_masked.gamp" + ] + ) + assert result == 1 + Path("masked.gamp").unlink(True) + + +def test_masking_errors_with_unmatching_data(): + result = masking.start_masking( + [ + "-i", "tests/test_data/docs/large.gamp", + "-m", "tests/test_data/docs/set2.pf", + "-o", "error_masked.gamp" + ] + ) + assert result == 1 + Path("masked.gamp").unlink(True) + + +def test_can_mask_data(): + result = masking.start_masking( + [ + "-i", "tests/test_data/docs/large.gamp", + "-m", "tests/test_data/docs/set1.pf", + "-o", "masked.gamp" + ] + ) + assert result != 1 + Path("masked.gamp").unlink() diff --git a/tests/test_full_program_execution.py b/tests/test_full_program_execution.py deleted file mode 100644 index b0a5125a..00000000 --- a/tests/test_full_program_execution.py +++ /dev/null @@ -1,43 +0,0 @@ -# coding=utf-8 -# -# PyPWA, a scientific analysis toolkit. -# Copyright (C) 2016 JLab -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -from pathlib import Path - -from PyPWA.progs import simulation -import pytest - -""" -Ensures that when given a simple dataset, the builtin programs are -capable of executing. -""" - - -@pytest.mark.skip() -def test_simulation_execute(): - output = Path("output.txt") - - simulation.simulation( - [ - "--disable_cache", "--output", "output.txt", - "config", "tests/test_data/docs/program_data/rho/RHOsim", - ] - ) - - if output.exists(): - output.unlink()