diff --git a/proMAD/config.py b/proMAD/config.py index a0ee45c..2d804a0 100644 --- a/proMAD/config.py +++ b/proMAD/config.py @@ -1,7 +1,7 @@ from pathlib import Path app_name = 'proMAD' -version_number = (0, 2, 5) +version_number = (0, 3, 0) version = f'{version_number[0]}.{version_number[1]}.{version_number[2]}' app_author = 'Anna Jaeschke; Hagen Eckert' url = 'https://proMAD.dev' @@ -9,6 +9,6 @@ base_dir = Path(__file__).absolute().parent array_data_folder = base_dir / 'data' / 'array' template_folder = base_dir / 'data' / 'templates' -allowed_load_version = (0, 1, 0) +allowed_load_version = (0, 3, 0) scale = 30 diff --git a/proMAD/core.py b/proMAD/core.py index 9d59296..1ca100d 100644 --- a/proMAD/core.py +++ b/proMAD/core.py @@ -99,6 +99,10 @@ def verbose_print(*args): self.backgrounds = [] self.foregrounds = [] self.bg_parameters = [] + self.original_average = [] + self.original_names = [] + self.original_index = [] + self.raw_index = [] self.exposure = [] self.meta_data = [] self.debug = None @@ -136,7 +140,8 @@ def verbose_print(*args): "is_finalized", "_kappa"] self.save_list_data = ['source_images', 'raw_images', 'backgrounds', 'foregrounds', - 'bg_parameters', 'exposure'] + 'bg_parameters', 'exposure', 'original_index', 'original_average', + 'original_average', 'raw_index'] self.grid_position = np.zeros((sum(self.array_data['net_layout_x']), sum(self.array_data['net_layout_y']), 2)) @@ -216,6 +221,10 @@ def list_types(cls): print(f'\tCompany: {array["company"]}') print(f'\tSource: {array["source"]}\n') + @property + def raw_names(self): + return [self.original_names[i] for i in self.raw_index] + def save(self, file): """ Saves the finalized content of an ArrayAnalyse instant into a .tar file @@ -244,6 +253,10 @@ def save(self, file): foregrounds=self.foregrounds, bg_parameters=self.bg_parameters, exposure=self.exposure, + original_names=self.original_names, + original_index=self.original_index, + original_average=self.original_average, + raw_index=self.raw_index, _fit_selection=self._fit_selection ) else: @@ -253,6 +266,10 @@ def save(self, file): backgrounds=self.backgrounds, foregrounds=self.foregrounds, bg_parameters=self.bg_parameters, + original_names=self.original_names, + original_index=self.original_index, + original_average=self.original_average, + raw_index=self.raw_index, ) if isinstance(file, os.PathLike) or isinstance(file, str): @@ -304,11 +321,13 @@ def load(cls, file): data = np.load(tar.extractfile(member)) if base_data is None or data is None: - warnings.warn("The loaded save file was not valid.", RuntimeWarning) - return None + tar.close() + raise TypeError("The loaded save file was not valid.") if not cls._compare_version(config.allowed_load_version, base_data['version']): - raise TypeError(f'Save file from version {".".join(base_data["version"])} cannot be loaded') + version_str = "{}.{}.{}".format(*base_data["version"]) + tar.close() + raise TypeError(f'A save file from version {version_str} cannot be loaded.') aa = cls(base_data['array_type'], silent=base_data['silent']) for name in aa.save_list_base: setattr(aa, name, base_data[name]) @@ -332,6 +351,11 @@ def reset_collection(self): self.bg_parameters = [] self.meta_data = [] self.exposure = [] + self.original_average = [] + self.original_index = [] + self.original_names = [] + self.raw_index = [] + self.original_names = [] self.is_finalized = False self.has_exposure = False @@ -381,6 +405,8 @@ def load_collection(self, data_input, rotation=None, finalize=True): if finalize and self.raw_images: self.finalize_collection() + else: + self.reset_collection() def finalize_collection(self): """ @@ -398,7 +424,10 @@ def finalize_collection(self): return None self.bg_parameters = np.array(self.bg_parameters) + self.raw_index = np.array(self.raw_index) + self.original_average = np.array(self.original_average) self.exposure = np.array(self.exposure) + if self.exposure.size == self.bg_parameters.size: order = np.argsort(self.exposure) self.exposure = self.exposure[order] @@ -406,6 +435,8 @@ def finalize_collection(self): else: order = np.argsort(self.bg_parameters) self.exposure = [] + + orginal_order = np.argsort(self.original_average) raw_images_array = np.zeros(shape=(self.raw_images[0].shape + (len(order),)), dtype=self.raw_images[0].dtype) backgrounds_array = np.zeros(shape=(self.backgrounds[0].shape + (len(order),)), @@ -419,22 +450,99 @@ def finalize_collection(self): backgrounds_array[:, :, n] = self.backgrounds[i] foregrounds_array[:, :, n] = self.foregrounds[i] source_images.append(self.source_images[i]) - meta = self.meta_data[i] + meta.append(self.meta_data[i]) self.raw_images = raw_images_array self.backgrounds = backgrounds_array self.foregrounds = foregrounds_array self.bg_parameters = self.bg_parameters[order] + self.raw_index = self.raw_index[order] + self.original_index = orginal_order + self.original_average = self.original_average[orginal_order] self.meta_data = meta self.source_images = source_images self.is_finalized = True if self.has_exposure: self.minimize_kappa() - if self.debug == 'plot': # pragma: no cover + if self.debug == 'plot': # pragma: no cover self.figure_alignment() self.figure_contact_sheet() - def load_image(self, file, rotation=None, suffix=None, meta_data=None): + def modify_exposure(self, exposure_info, test=False): + """ + Add or modify exposure information of a finalized collection. + + Notes + ----- + In case set exposure time changes the order + + Parameters + ---------- + exposure_info: Union[dict, list] + either list of exposure times with the same length and order as shown by *raw_names* or + dict describing start and step size of the exposure `{'start': 10, 'step': 30}` + unit in seconds + test: bool + if True no changes are made + + Returns + ------- + exposure: list + generated list of exposure in order of *raw_names* + reorder: bool + if the new exposure implies a reordering True is returned + + """ + + if not self.is_finalized: + warnings.warn('Data collection needs to be finalized to amend exposure information.', RuntimeWarning) + return None + if isinstance(exposure_info, dict): + if 'start' in exposure_info and 'step' in exposure_info: + raw_exposure = exposure_info['start'] + np.array(range(len(self.original_average))) * exposure_info['step'] + raw_exposure_lookup = {i: raw_exposure[n] for n, i in enumerate(self.original_index)} + exposure = [raw_exposure_lookup[i] for i in self.raw_index] + else: + warnings.warn('Exposure information needs the keys "start" and "step".', RuntimeWarning) + return None, None + else: + if len(exposure_info) == len(self.bg_parameters): + exposure = exposure_info + else: + warnings.warn('Exposure information has the wrong length.', RuntimeWarning) + return None, None + + order = np.argsort(exposure) + reorder = not np.all(order[:-1] <= order[1:]) + if test: + return exposure, reorder + + self.exposure = exposure + if reorder: + raw_images_array = np.zeros_like(self.raw_images) + backgrounds_array = np.zeros_like(self.backgrounds) + foregrounds_array = np.zeros_like(self.foregrounds) + source_images = [] + meta = [] + for n, i in enumerate(order): + raw_images_array[:, :, n] = self.raw_images[:, :, i] + backgrounds_array[:, :, n] = self.backgrounds[:, :, i] + foregrounds_array[:, :, n] = self.foregrounds[:, :, i] + source_images.append(self.source_images[i]) + meta.append(self.meta_data[i]) + self.raw_images = raw_images_array + self.backgrounds = backgrounds_array + self.foregrounds = foregrounds_array + self.bg_parameters = self.bg_parameters[order] + self.raw_index = self.raw_index[order] + self.meta_data = meta + self.source_images = source_images + + self.has_exposure = True + self.minimize_kappa() + return self.exposure, reorder + + def load_image(self, file, rotation=None, filename=None, meta_data=None): """ Load a single image file into the collection. @@ -446,10 +554,11 @@ def load_image(self, file, rotation=None, suffix=None, meta_data=None): if file is None result is directly shown rotation: int or float apply a rotation to the images - suffix: str - if a file-like object is submitted the suffix is needed for type identification (".tif", ".png", ...) + filename: str + if a file-like object is submitted the filename is needed for type identification (".tif", ".png", ...) meta_data: dict() + """ if self.is_finalized: @@ -461,7 +570,7 @@ def load_image(self, file, rotation=None, suffix=None, meta_data=None): self.verbose_print(f'Load image: {file}') file = Path(file) source_image = ski_io.imread(file.absolute(), plugin='imageio') - suffix = file.suffix.lower() + filename = file.name elif isinstance(file, np.ndarray): source_image = file elif isinstance(file, (io.RawIOBase, io.BufferedIOBase)): @@ -472,7 +581,7 @@ def load_image(self, file, rotation=None, suffix=None, meta_data=None): return None if meta_data is None: - if suffix == '.tif': + if filename.lower().endswith('.tif'): if file_system: with tifffile.TiffFile(str(file.absolute())) as tif_data: tags = [page.tags for page in tif_data.pages] @@ -497,6 +606,8 @@ def load_image(self, file, rotation=None, suffix=None, meta_data=None): source_image = rotate(source_image, rotation, resize=True) source_image = img_as_float(source_image) + self.original_average.append(np.average(source_image)) + self.original_names.append(filename) raw_image = self.warp_image(source_image, rotation=rotation) if raw_image is not None: @@ -529,6 +640,7 @@ def load_image(self, file, rotation=None, suffix=None, meta_data=None): self.foregrounds.append(image) self.bg_parameters.append(self.background_histogram(raw_image)) self.meta_data.append(meta_data) + self.raw_index.append(len(self.original_average) - 1) if meta_data: if 'exposure_time' in meta_data: self.exposure.append(meta_data['exposure_time']) diff --git a/setup.py b/setup.py index 0e23dbf..557bb18 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ keywords=['protein', 'microarrays', 'densitometric'], python_requires='~=3.6', classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Intended Audience :: Science/Research', 'Topic :: Scientific/Engineering :: Bio-Informatics', @@ -43,6 +43,7 @@ 'Operating System :: OS Independent', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3 :: Only' ], ) diff --git a/tests/cases/save/dump.tar b/tests/cases/save/dump_0_1_0.tar similarity index 100% rename from tests/cases/save/dump.tar rename to tests/cases/save/dump_0_1_0.tar diff --git a/tests/cases/save/dump_0_3_0.tar b/tests/cases/save/dump_0_3_0.tar new file mode 100644 index 0000000..68ac9f0 Binary files /dev/null and b/tests/cases/save/dump_0_3_0.tar differ diff --git a/tests/test_core.py b/tests/test_core.py index a4e6915..2b6a3f8 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -4,6 +4,7 @@ import warnings from pathlib import Path +import numpy as np from proMAD import ArrayAnalyse from helper import hash_file, hash_mem, hash_array @@ -47,14 +48,22 @@ def tearDownClass(cls): class LoadFromFile(unittest.TestCase): + cases = Path(__file__).absolute().resolve().parent / 'cases' - def test_load(self): - cases = Path(__file__).absolute().resolve().parent / 'cases' - aa = ArrayAnalyse.load(cases / 'save' / 'dump.tar') - self.assertEqual(hash_array(aa.foregrounds), - '993f84db0f1211cfd9859571e9d1db8dc2443d179c5199c60fd3774057f27f0f') - self.assertEqual(hash_array(aa.raw_images), - '46ee47b580e20c10cd9c50c598944929887f41a86f64ecca8071085ae5dde93c') + def test_load_too_old(self): + self.assertRaises(TypeError, ArrayAnalyse.load, self.cases / 'save' / 'dump_0_1_0.tar') + + def test_modify_exposure_t(self): + aa = ArrayAnalyse.load(self.cases / 'save' / 'dump_0_3_0.tar') + exposure, reorder = aa.modify_exposure({'start': 10, 'step': 30}, test=True) + self.assertFalse(reorder) + self.assertListEqual(exposure, [10, 40, 70, 100, 130]) + + def test_modify_exposure_mix(self): + aa = ArrayAnalyse.load(self.cases / 'save' / 'dump_0_3_0.tar') + exposure, reorder = aa.modify_exposure([10, 40, 100, 70, 130]) + self.assertTrue(reorder) + np.testing.assert_array_equal(aa.raw_index, [0, 1, 3, 2, 4]) class TestArrays(unittest.TestCase): @@ -134,10 +143,10 @@ def test_load_image(self): content = (self.cases / 'prepared/prepared_00030.tif').read_bytes() mem_im = io.BytesIO(content) - self.aa.load_image(mem_im, rotation=90, suffix='.tif') + self.aa.load_image(mem_im, rotation=90, filename='prepared/prepared_00030.tif') with (self.cases / 'prepared/prepared_00032.tif').open('rb') as fo: - self.aa.load_image(fo, rotation=90, suffix='.tif') + self.aa.load_image(fo, rotation=90, filename='prepared/prepared_00032.tif') self.aa.finalize_collection() @@ -161,6 +170,7 @@ def test_load_image(self): self.assertEqual(len(w), 1) self.assertEqual(w[-1].category, RuntimeWarning) self.assertIn("Data is already finalized.", str(w[-1].message)) + self.aa.reset_collection() self.assertEqual(self.aa.source_images, []) @@ -324,14 +334,21 @@ def test_figure_reaction_fit(self): self.assertEqual(hash_file(self.out_folder / 'reaction_fit.png'), '12e95aac956289f7f24b46e5a1ee3e4149aa3a869602bdeff7fa407bab565bab') - def test_save(self): + def test_save_load(self): save_mem = io.BytesIO() - hash_compare = ['e0ba8b1fea5c9b2dab4cabcff8447bb27fa46bba0be766fe12950c790023242a', # python 3.8 - 'eff29a8d43c76e929d09e90dc7f1cbf2679d5ea73fc5762b3d71ff65c4a29f54'] self.aa.save(file=save_mem) - self.assertIn(hash_mem(save_mem), hash_compare) + save_mem_hash = hash_mem(save_mem, skip=0) + save_mem.seek(0) + saved_aa = ArrayAnalyse.load(save_mem) + np.testing.assert_array_equal(saved_aa.original_index, self.aa.original_index) + np.testing.assert_array_equal(saved_aa.exposure, self.aa.exposure) + np.testing.assert_array_equal(saved_aa.raw_images, self.aa.raw_images) + del saved_aa + self.aa.save(self.out_folder / 'dump.tar') - self.assertIn(hash_file(self.out_folder / 'dump.tar'), hash_compare) + save_file_hash = hash_file(self.out_folder / 'dump.tar', skip=0) + + self.assertEqual(save_file_hash, save_mem_hash) if __name__ == '__main__': diff --git a/tests/test_report.py b/tests/test_report.py index f0ee556..97955ee 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -15,7 +15,7 @@ class TestWithARY022B(unittest.TestCase): @classmethod def setUpClass(cls): cls.cases = Path(__file__).absolute().resolve().parent / 'cases' - cls.aa = ArrayAnalyse.load(cls.cases / 'save' / 'dump.tar') + cls.aa = ArrayAnalyse.load(cls.cases / 'save' / 'dump_0_3_0.tar') cls.out_folder = cls.cases / 'testing_reports' cls.out_folder.mkdir(exist_ok=True, parents=True) cls.additional_info = [