Inclusion of IROLL, IPCA, NEGFD routines and new features to NEGFC routines

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# This workflow will install Python depe:ndencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: ci
@@ -24,14 +24,29 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest pytest-cov sphinx
+        python -m pip install pre-commit pytest pytest-cov sphinx
         if [ -f requirements.txt ]; then pip install -r requirements-dev.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+#    Uncomment below when ready to deal with a lot of PEP8 formatting changes
+#    - name: Verify files with pre-commit
+#      run: |
+#        # Setup pre-commit hooks
+#        pre-commit clean
+#        pre-commit install --hook-type pre-merge-commit
+#        pre-commit install --hook-type pre-push
+#        pre-commit install --hook-type post-rewrite
+#        pre-commit install-hooks
+#        pre-commit install
+#        # Run pre-commit hooks
+#        pre-commit run --files vip_hci/config/*.py
+#	 pre-commit run --files vip_hci/fits/*.py
+#	 pre-commit run --files vip_hci/fm/*.py
+#	 pre-commit run --files vip_hci/greedy/*.py
+#	 pre-commit run --files vip_hci/invprob/*.py
+#	 pre-commit run --files vip_hci/metrics/*.py
+#        pre-commit run --files vip_hci/preproc/*.py
+#        pre-commit run --files vip_hci/psfsub/*.py
+#        pre-commit run --files vip_hci/stats/*.py
+#        pre-commit run --files vip_hci/var/*.py
     - name: Test with pytest
       run: |
         pytest tests/pre_3_10 --cov=vip_hci/ --cov-report=xml --splits 3 --group ${{ matrix.group }}

Makefile

@@ -47,6 +47,16 @@ test:
 	pre-commit install --hook-type post-rewrite
 	pre-commit install-hooks
 	pre-commit install
+	pre-commit run --files vip_hci/config/*.py
+	pre-commit run --files vip_hci/fits/*.py
+	pre-commit run --files vip_hci/fm/*.py
+	pre-commit run --files vip_hci/greedy/*.py
+	pre-commit run --files vip_hci/invprob/*.py
+	pre-commit run --files vip_hci/metrics/*.py
+	pre-commit run --files vip_hci/preproc/*.py
+	pre-commit run --files vip_hci/psfsub/*.py
+	pre-commit run --files vip_hci/stats/*.py
+	pre-commit run --files vip_hci/var/*.py
 	pre-commit run --files vip_hci/objects/*.py
 	pytest --cov=vip_hci/ --cov-report=xml
 	rm confi_hist.pdf

requirements.txt

@@ -3,7 +3,7 @@ scipy
 astropy
 photutils
 scikit-learn
-scikit-image
+scikit-image>0.17.0, <=0.18.3 # unsolved bug in skimage.registration.phase_cross_correlation for versions >0.18.3 up to at least 0.22. Req>0.17 to have the reference mask option implemented in that function.
 emcee==2.2.1
 nestle
 corner
@@ -15,4 +15,4 @@ munch
 nbsphinx
 hciplot>=0.2.4
 typing
-dataclass_builder
+dataclass_builder

tests/conftest.py

@@ -25,17 +25,17 @@ def example_dataset_adi():
     -----
     We use the helper function ``download_resource`` which handles the request
     and puts it to sleep for a defined duration if too many requests are done.
-    They inherently call the Astropy's ``download_file`` function which uses caching,
-    so the file is downloaded at most once per test run.
+    They inherently call the Astropy's ``download_file`` function which uses
+    caching, so the file is downloaded at most once per test run.
 
     """
     print("downloading data...")
 
-    url_prefix = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
+    url_d = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
 
-    f1 = download_resource("{}/naco_betapic_cube_cen.fits".format(url_prefix))
-    f2 = download_resource("{}/naco_betapic_psf.fits".format(url_prefix))
-    f3 = download_resource("{}/naco_betapic_pa.fits".format(url_prefix))
+    f1 = download_resource("{}/naco_betapic_cube_cen.fits".format(url_d))
+    f2 = download_resource("{}/naco_betapic_psf.fits".format(url_d))
+    f3 = download_resource("{}/naco_betapic_pa.fits".format(url_d))
 
     # load fits
     cube = vip.fits.open_fits(f1)
@@ -93,18 +93,18 @@ def example_dataset_ifs():
     -----
     We use the helper function ``download_resource`` which handles the request
     and puts it to sleep for a defined duration if too many requests are done.
-    They inherently call the Astropy's ``download_file`` function which uses caching,
-    so the file is downloaded at most once per test run.
+    They inherently call the Astropy's ``download_file`` function which uses
+    caching, so the file is downloaded at most once per test run.
 
     """
     print("downloading data...")
 
-    url_prefix = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
+    url_d = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
 
-    f1 = download_resource("{}/sphere_v471tau_cube.fits".format(url_prefix))
-    f2 = download_resource("{}/sphere_v471tau_psf.fits".format(url_prefix))
-    f3 = download_resource("{}/sphere_v471tau_pa.fits".format(url_prefix))
-    f4 = download_resource("{}/sphere_v471tau_wl.fits".format(url_prefix))
+    f1 = download_resource("{}/sphere_v471tau_cube.fits".format(url_d))
+    f2 = download_resource("{}/sphere_v471tau_psf.fits".format(url_d))
+    f3 = download_resource("{}/sphere_v471tau_pa.fits".format(url_d))
+    f4 = download_resource("{}/sphere_v471tau_wl.fits".format(url_d))
 
     # load fits
     cube = vip.fits.open_fits(f1)
@@ -134,7 +134,7 @@ def example_dataset_ifs():
 @pytest.fixture(scope="session")
 def example_dataset_ifs_crop():
     """
-    Download example FITS cube from github + prepare HCIDataset object, after
+    Download example FITS cube from github + prepare HCIDataset object, after\
     cropping to only 3 sp. channels (faster NEGFC test).
 
     Returns
@@ -145,18 +145,18 @@ def example_dataset_ifs_crop():
     -----
     We use the helper function ``download_resource`` which handles the request
     and puts it to sleep for a defined duration if too many requests are done.
-    They inherently call the Astropy's ``download_file`` function which uses caching,
-    so the file is downloaded at most once per test run.
+    They inherently call the Astropy's ``download_file`` function which uses
+    caching, so the file is downloaded at most once per test run.
 
     """
     print("downloading data...")
 
-    url_prefix = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
+    url_d = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
 
-    f1 = download_resource("{}/sphere_v471tau_cube.fits".format(url_prefix))
-    f2 = download_resource("{}/sphere_v471tau_psf.fits".format(url_prefix))
-    f3 = download_resource("{}/sphere_v471tau_pa.fits".format(url_prefix))
-    f4 = download_resource("{}/sphere_v471tau_wl.fits".format(url_prefix))
+    f1 = download_resource("{}/sphere_v471tau_cube.fits".format(url_d))
+    f2 = download_resource("{}/sphere_v471tau_psf.fits".format(url_d))
+    f3 = download_resource("{}/sphere_v471tau_pa.fits".format(url_d))
+    f4 = download_resource("{}/sphere_v471tau_wl.fits".format(url_d))
 
     # load fits
     cube = vip.fits.open_fits(f1)
@@ -199,17 +199,17 @@ def example_dataset_rdi():
     -----
     We use the helper function ``download_resource`` which handles the request
     and puts it to sleep for a defined duration if too many requests are done.
-    They inherently call the Astropy's ``download_file`` function which uses caching,
-    so the file is downloaded at most once per test run.
+    They inherently call the Astropy's ``download_file`` function which uses
+    caching, so the file is downloaded at most once per test run.
 
     """
     print("downloading data...")
 
-    url_prefix = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
+    url_d = "https://github.com/vortex-exoplanet/VIP_extras/raw/master/datasets"
 
-    f1 = download_resource("{}/naco_betapic_cube_cen.fits".format(url_prefix))
-    f2 = download_resource("{}/naco_betapic_psf.fits".format(url_prefix))
-    f3 = download_resource("{}/naco_betapic_pa.fits".format(url_prefix))
+    f1 = download_resource("{}/naco_betapic_cube_cen.fits".format(url_d))
+    f2 = download_resource("{}/naco_betapic_psf.fits".format(url_d))
+    f3 = download_resource("{}/naco_betapic_pa.fits".format(url_d))
 
     # load fits
     cube = vip.fits.open_fits(f1)
@@ -255,7 +255,7 @@ def example_dataset_rdi():
 
 @pytest.fixture(autouse=True)
 def time_test():
-    """Time a test and print out how long it took"""
+    """Time a test and print out how long it took."""
     before = time.time()
     yield
     after = time.time()
@@ -264,7 +264,7 @@ def time_test():
 
 @pytest.fixture(autouse=True, scope="session")
 def time_all_tests():
-    """Time a test and print out how long it took"""
+    """Time all tests and print out how long they took."""
     before = time.time()
     yield
     after = time.time()

tests/post_3_10/test_objects_pppca.py

@@ -1,14 +1,12 @@
 """Test for the PostProc object dedicated to principal component analysis."""
 
-__author__ = "Thomas Bédrine"
+__author__ = "Carlos Gomez Gonzalez, Thomas Bédrine"
 
 import copy
 
 import numpy as np
 from dataclass_builder import update
-
 import sys
-
 sys.path.append(".../tests")
 from tests.helpers import fixture, check_detection
 from vip_hci.objects import PCABuilder
@@ -17,8 +15,7 @@
 NO_FRAME_CASE = ["pca_drot", "pca_ann_auto"]
 PREV_CASE = ["pca_grid_list"]
 
-# Note : this function comes from the former test for adi psfsub, I did not write it,
-# and I didn't found the author (feel free to add the author if you know them)
+
 @fixture(scope="module")
 def injected_cube_position(example_dataset_adi):
     """

tests/pre_3_10/test_fm_fakedisk.py

@@ -45,7 +45,7 @@ def _expected():
         """
         Expected positions.
         """
-        return [(15, 12), (12, 9), (9, 12)]
+        return [(15, 12), (12, 15), (9, 12)]
 
     psf = np.zeros((25, 25))
     psf[15, 12] = 1

tests/pre_3_10/test_fm_negfc_3d.py

@@ -67,7 +67,7 @@ def injected_cube_position(example_dataset_adi):
     "pca_algo, negfc_algo, ncomp, mu_sigma, fm, force_rpa, conv_test",
     [
         (pca_annular, firstguess, 3, False, "stddev", False, None),
-        (pca, firstguess, 3, True, None, False, None),
+        (pca, firstguess, 5, True, None, False, None),
         (median_sub, firstguess, None, False, "sum", False, None),
         (pca_annulus, mcmc_negfc_sampling, 3, False, "stddev", False, "gb"),
         (pca_annulus, mcmc_negfc_sampling, 3, True, None, True, "ac"),
@@ -88,9 +88,10 @@ def test_algos(
 
     # run firstguess with simplex only if followed by mcmc or nested sampling
     if pca_algo == median_sub:
-        algo_options = {"imlib": "opencv", "verbose": False}
+        algo_options = {"imlib": "opencv", "interpolation": "lanczos4",
+                        "verbose": False}
     else:
-        algo_options = {"imlib": "opencv"}
+        algo_options = {"imlib": "opencv", "interpolation": "lanczos4"}
     res0 = firstguess(
         cube=ds.cube,
         angs=ds.angles,
@@ -112,8 +113,9 @@ def test_algos(
 
     if negfc_algo == firstguess:
         # use injection of 180 companions in empty cube to estimate error bars
-        cube_emp = cube_planet_free(res, ds.cube, ds.angles, ds.psf, imlib="opencv")
-        algo_options = {"imlib": "opencv"}
+        cube_emp = cube_planet_free(res, ds.cube, ds.angles, ds.psf,
+                                    imlib="opencv", interpolation="lanczos4")
+        algo_options = {"imlib": "opencv", "interpolation": "lanczos4"}
         if pca_algo != median_sub:
             algo_options["ncomp"] = ncomp
         if pca_algo == pca_annular:
@@ -125,7 +127,7 @@ def test_algos(
             sp_unc = speckle_noise_uncertainty(
                 cube_emp,
                 res,
-                np.arange(0, 360, 3),
+                np.arange(0, 360, 2),
                 ds.angles,
                 algo=pca_algo,
                 psfn=ds.psf,
@@ -137,6 +139,7 @@ def test_algos(
                 full_output=False,
                 algo_options=algo_options,
                 nproc=1,
+                sigma_trim=3
             )
         else:
             sp_unc = (2, 2, 0.1 * gt[2])
@@ -167,6 +170,7 @@ def test_algos(
             sigma="spe+pho",
             fmerit=fm,
             imlib="opencv",
+            interpolation="lanczos4",
             nwalkers=100,
             niteration_min=200,
             niteration_limit=niteration_limit,
@@ -245,7 +249,7 @@ def test_algos(
             decline_factor=None,
             rstate=None,
             verbose=True,
-            algo_options={"imlib": "opencv"},
+            algo_options={"imlib": "opencv", "interpolation": "lanczos4"},
         )
         # infer mu, sigma from nested sampling result
         mu_sig = nested_sampling_results(res, burnin=0.3, bins=None, save=False)

tests/pre_3_10/test_fm_negfc_4d.py

@@ -76,6 +76,7 @@ def test_algos(injected_cube_position, pca_algo, negfc_algo, ncomp, mu_sigma, fm
         fmerit=fm,
         mu_sigma=mu_sigma,
         imlib="opencv",
+        interpolation='lanczos4',
         aperture_radius=2,
         annulus_width=4 * fwhm_m,
     )

tests/pre_3_10/test_metrics_contrcurve.py

@@ -2,17 +2,19 @@
 Tests for metrics/contrcurve.py
 
 """
-from vip_hci.psfsub import pca, PCA_Params
-from vip_hci.preproc import frame_crop
-from vip_hci.metrics import contrast_curve
-from vip_hci.fm.utils_negfc import find_nearest
-from vip_hci.fm.utils_negfc import cube_planet_free
-from vip_hci.config import VLT_NACO
-from tests.helpers import np
-from tests.helpers import fixture
 import copy
 import sys
 
+from tests.helpers import fixture
+from tests.helpers import np
+from vip_hci.config import VLT_NACO
+from vip_hci.fm.utils_negfc import cube_planet_free
+from vip_hci.fm.utils_negfc import find_nearest
+from vip_hci.metrics import contrast_curve
+from vip_hci.preproc import frame_crop
+from vip_hci.psfsub import pca
+from vip_hci.psfsub import PCA_Params
+
 sys.path.append(".../tests")
 
 sys.path.append(".../tests")
@@ -99,26 +101,26 @@ def test_contrast_curve(get_cube):
     msg = "Contrast too shallow compared to expectations: {} > {}"
     assert check, msg.format(cc_stu, 4e-3 * corr_r)
 
-    # check that at 0.4'' 5-sigma cc < 4e-4
+    # check that at 0.4'' 5-sigma cc < 5e-4
     idx_r = find_nearest(rad * plsc, 0.4)
     cc_gau = gauss_cc[idx_r]
     corr_r = sigma_corr[idx_r]
 
-    if cc_gau < 4e-4:
+    if cc_gau < 5e-4:
         check = True
     else:
         check = False
     msg = "Contrast too shallow compared to expectations: {} > {}"
-    assert check, msg.format(cc_gau, 4e-4)
+    assert check, msg.format(cc_gau, 5e-4)
 
     # check that at 0.4'' 5-sigma cc: Student statistics > Gaussian statistics
     cc_stu = student_cc[idx_r]
-    if cc_stu < 4e-4 * corr_r and cc_stu > cc_gau:
+    if cc_stu < 5e-4 * corr_r and cc_stu > cc_gau:
         check = True
-    elif cc_stu < 4e-4 * corr_r:
+    elif cc_stu < 5e-4 * corr_r:
         check = False
         msg = "Student-statistics cc smaller than Gaussian statistics cc"
     else:
         check = False
         msg = "Contrast too shallow compared to expectations: {} > {}"
-    assert check, msg.format(cc_stu, 4e-4 * corr_r)
+    assert check, msg.format(cc_stu, 5e-4 * corr_r)