notebooks/st_align_Level3_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ae6bd1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JWST version: 1.8.0\n",
      "Found 4 input files, checking for the right filter...\n",
      "KEEP ./test_mosaic/F560W_level2_gaia/jw02666001001_02101_00001_mirimage_tweakregstep.fits: F560W\n",
      "KEEP ./test_mosaic/F560W_level2_gaia/jw02666001001_02101_00002_mirimage_tweakregstep.fits: F560W\n",
      "KEEP ./test_mosaic/F560W_level2_gaia/jw02666001001_02101_00003_mirimage_tweakregstep.fits: F560W\n",
      "KEEP ./test_mosaic/F560W_level2_gaia/jw02666001001_02101_00004_mirimage_tweakregstep.fits: F560W\n",
      "4 out of 4 have the correct filter\n",
      "Outdir level3: ./test_mosaic/F560W_level3_gaia\n"
     ]
    }
   ],
   "source": [
    "import glob,os\n",
    "import jwst\n",
    "from astropy.io import fits\n",
    "print(f'JWST version: {jwst.__version__}')\n",
    "\n",
    "root_dir = './test_mosaic'\n",
    "filter_name = 'F560W'\n",
    "\n",
    "# input dir and files\n",
    "inputdir_level3 = f'{root_dir}/{filter_name}_level2_gaia'\n",
    "filepattern = '*tweakregstep.fits'\n",
    "inputfiles_level3 = sorted(glob.glob(f'{inputdir_level3}/{filepattern}'))\n",
    "print(f'Found {len(inputfiles_level3)} input files, checking for the right filter...')\n",
    "# check for correct filter\n",
    "inputfiles_level3_filter = []\n",
    "for filename in inputfiles_level3: \n",
    "    hdr = fits.getheader(filename)\n",
    "    if hdr[\"FILTER\"].lower() == filter_name.lower():\n",
    "        print(f'KEEP {filename}: {hdr[\"FILTER\"]}')\n",
    "        inputfiles_level3_filter.append(os.path.abspath(filename))\n",
    "    else:\n",
    "        print(f'skipping {filename}: {hdr[\"FILTER\"]}')\n",
    "print(f'{len(inputfiles_level3_filter)} out of {len(inputfiles_level3)} have the correct filter')\n",
    "\n",
    "# output dir, and make sure it exists\n",
    "outdir_level3 = f'{root_dir}/{filter_name}_level3_gaia'\n",
    "print(f'Outdir level3: {outdir_level3}')\n",
    "if not os.path.isdir(outdir_level3):\n",
    "    os.makedirs(outdir_level3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "aea17de5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jwnoprogram-a3001_none_003_asn with 1 products\n",
      "Rule=DMS_Level3_Base\n",
      "No constraints\n",
      "Products:\n",
      "\tF560W with 4 members\n"
     ]
    }
   ],
   "source": [
    "from jwst.associations import asn_from_list\n",
    "from jwst.associations.lib.rules_level3_base import DMS_Level3_Base\n",
    "\n",
    "asn3 = asn_from_list.asn_from_list(inputfiles_level3_filter, rule=DMS_Level3_Base, product_name=filter_name)\n",
    "miri_asn_file = f'{outdir_level3}/{filter_name}.json'\n",
    "with open(miri_asn_file, 'w') as outfile:\n",
    "    name, serialized = asn3.dump(format='json')\n",
    "    outfile.write(serialized)\n",
    "print(asn3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a22b1c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:55:30,061 - stpipe.Image3Pipeline - INFO - Image3Pipeline instance created.\n",
      "2022-12-14 12:55:30,062 - stpipe.Image3Pipeline.assign_mtwcs - INFO - AssignMTWcsStep instance created.\n",
      "2022-12-14 12:55:30,064 - stpipe.Image3Pipeline.tweakreg - INFO - TweakRegStep instance created.\n",
      "2022-12-14 12:55:30,066 - stpipe.Image3Pipeline.skymatch - INFO - SkyMatchStep instance created.\n",
      "2022-12-14 12:55:30,069 - stpipe.Image3Pipeline.outlier_detection - INFO - OutlierDetectionStep instance created.\n",
      "2022-12-14 12:55:30,072 - stpipe.Image3Pipeline.resample - INFO - ResampleStep instance created.\n",
      "2022-12-14 12:55:30,074 - stpipe.Image3Pipeline.source_catalog - INFO - SourceCatalogStep instance created.\n"
     ]
    }
   ],
   "source": [
    "from jwst.pipeline import calwebb_image3\n",
    "\n",
    "image3 = calwebb_image3.Image3Pipeline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d432a3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "image3.output_dir = outdir_level3\n",
    "image3.save_results = True\n",
    "### IMPORTANT!! you have to set image3.tweakreg.skip = True, otherwise tweakreg will screw up the WCS again!\n",
    "image3.tweakreg.skip = True\n",
    "#image3.tweakreg.align_to_gaia = True\n",
    "image3.skymatch.skip = True\n",
    "#image3.skymatch.skymethod = 'global+match'\n",
    "image3.skymatch.match_down = False\n",
    "image3.source_catalog.skip=False\n",
    "#image3.skymatch.subtract = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "90fd9169",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:55:37,077 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline running with args ('./test_mosaic/F560W_level3_gaia/F560W.json',).\n",
      "2022-12-14 12:55:37,089 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'steps': {'assign_mtwcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'assign_mtwcs', 'search_output_file': True, 'input_dir': ''}, 'tweakreg': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}, 'skymatch': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'skymethod': 'match', 'match_down': False, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}, 'outlier_detection': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}, 'resample': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'source_catalog': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}}}\n",
      "2022-12-14 12:55:38,108 - stpipe.Image3Pipeline - WARNING - /Users/arest/anaconda3/envs/jwst/lib/python3.10/site-packages/jwst/associations/association.py:215: UserWarning: 'expname' contains path, but should only be a filename.  All input files should be in a single directory, so no path is needed.\n",
      "  warnings.warn(err_str, UserWarning)\n",
      "\n",
      "2022-12-14 12:55:38,812 - stpipe.Image3Pipeline - INFO - Prefetching reference files for dataset: 'jw02666001001_02101_00001_mirimage_tweakregstep.fits' reftypes = ['abvegaoffset', 'apcorr', 'drizpars']\n",
      "2022-12-14 12:55:43,585 - stpipe.Image3Pipeline - INFO - Prefetch for ABVEGAOFFSET reference file is '/grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf'.\n",
      "2022-12-14 12:55:43,621 - stpipe.Image3Pipeline - INFO - Prefetch for APCORR reference file is '/grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits'.\n",
      "2022-12-14 12:55:43,658 - stpipe.Image3Pipeline - INFO - Prefetch for DRIZPARS reference file is '/grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits'.\n",
      "2022-12-14 12:55:43,698 - stpipe.Image3Pipeline - INFO - Starting calwebb_image3 ...\n",
      "2022-12-14 12:55:43,716 - stpipe.Image3Pipeline - WARNING - /Users/arest/anaconda3/envs/jwst/lib/python3.10/site-packages/jwst/associations/association.py:215: UserWarning: 'expname' contains path, but should only be a filename.  All input files should be in a single directory, so no path is needed.\n",
      "  warnings.warn(err_str, UserWarning)\n",
      "\n",
      "2022-12-14 12:55:45,786 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg running with args (<ModelContainer>,).\n",
      "2022-12-14 12:55:45,789 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': './test_mosaic/F560W_level3_gaia', 'save_catalogs': False, 'catalog_format': 'ecsv', 'kernel_fwhm': 2.5, 'snr_threshold': 10.0, 'sharplo': 0.2, 'sharphi': 1.0, 'roundlo': -1.0, 'roundhi': 1.0, 'brightest': 200, 'peakmax': None, 'bkg_boxsize': 400, 'enforce_user_order': False, 'expand_refcat': False, 'minobj': 15, 'searchrad': 2.0, 'use2dhist': True, 'separation': 1.0, 'tolerance': 0.7, 'xoffset': 0.0, 'yoffset': 0.0, 'fitgeometry': 'rshift', 'nclip': 3, 'sigma': 3.0, 'abs_refcat': '', 'save_abs_catalog': False, 'abs_minobj': 15, 'abs_searchrad': 6.0, 'abs_use2dhist': True, 'abs_separation': 0.1, 'abs_tolerance': 0.7, 'abs_fitgeometry': 'rshift', 'abs_nclip': 3, 'abs_sigma': 3.0}\n",
      "2022-12-14 12:55:45,790 - stpipe.Image3Pipeline.tweakreg - INFO - Step skipped.\n",
      "2022-12-14 12:55:45,805 - stpipe.Image3Pipeline.tweakreg - INFO - Step tweakreg done\n",
      "2022-12-14 12:55:45,884 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch running with args (<ModelContainer>,).\n",
      "2022-12-14 12:55:45,887 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': './test_mosaic/F560W_level3_gaia', 'skymethod': 'match', 'match_down': False, 'subtract': False, 'stepsize': None, 'skystat': 'mode', 'dqbits': '~DO_NOT_USE+NON_SCIENCE', 'lower': None, 'upper': None, 'nclip': 5, 'lsigma': 4.0, 'usigma': 4.0, 'binwidth': 0.1}\n",
      "2022-12-14 12:55:45,887 - stpipe.Image3Pipeline.skymatch - INFO - Step skipped.\n",
      "2022-12-14 12:55:45,902 - stpipe.Image3Pipeline.skymatch - INFO - Step skymatch done\n",
      "2022-12-14 12:55:45,983 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection running with args (<ModelContainer>,).\n",
      "2022-12-14 12:55:45,985 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'crf', 'search_output_file': False, 'input_dir': './test_mosaic/F560W_level3_gaia', 'weight_type': 'ivm', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'nlow': 0, 'nhigh': 0, 'maskpt': 0.7, 'grow': 1, 'snr': '5.0 4.0', 'scale': '1.2 0.7', 'backg': 0.0, 'save_intermediate_results': False, 'resample_data': True, 'good_bits': '~DO_NOT_USE', 'scale_detection': False, 'allowed_memory': None, 'in_memory': False}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:55:45,995 - stpipe.Image3Pipeline.outlier_detection - INFO - Performing outlier detection on 4 inputs\n",
      "2022-12-14 12:55:45,995 - stpipe.Image3Pipeline.outlier_detection - INFO - Driz parameter kernel: square\n",
      "2022-12-14 12:55:45,996 - stpipe.Image3Pipeline.outlier_detection - INFO - Driz parameter pixfrac: 1.0\n",
      "2022-12-14 12:55:45,997 - stpipe.Image3Pipeline.outlier_detection - INFO - Driz parameter fillval: INDEF\n",
      "2022-12-14 12:55:45,997 - stpipe.Image3Pipeline.outlier_detection - INFO - Driz parameter weight_type: ivm\n",
      "2022-12-14 12:55:45,998 - stpipe.Image3Pipeline.outlier_detection - INFO - Output pixel scale ratio: 1.0\n",
      "2022-12-14 12:55:46,185 - stpipe.Image3Pipeline.outlier_detection - INFO - 1 exposures to drizzle together\n",
      "2022-12-14 12:55:46,774 - stpipe.Image3Pipeline.outlier_detection - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:55:47,129 - stpipe.Image3Pipeline.outlier_detection - INFO - Exposure jw02666001001_02101_00001_mirimage_outlier_i2d.fits saved to file\n",
      "2022-12-14 12:55:47,136 - stpipe.Image3Pipeline.outlier_detection - INFO - 1 exposures to drizzle together\n",
      "2022-12-14 12:55:47,693 - stpipe.Image3Pipeline.outlier_detection - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:55:48,025 - stpipe.Image3Pipeline.outlier_detection - INFO - Exposure jw02666001001_02101_00002_mirimage_outlier_i2d.fits saved to file\n",
      "2022-12-14 12:55:48,030 - stpipe.Image3Pipeline.outlier_detection - INFO - 1 exposures to drizzle together\n",
      "2022-12-14 12:55:48,598 - stpipe.Image3Pipeline.outlier_detection - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:55:48,922 - stpipe.Image3Pipeline.outlier_detection - INFO - Exposure jw02666001001_02101_00003_mirimage_outlier_i2d.fits saved to file\n",
      "2022-12-14 12:55:48,927 - stpipe.Image3Pipeline.outlier_detection - INFO - 1 exposures to drizzle together\n",
      "2022-12-14 12:55:49,548 - stpipe.Image3Pipeline.outlier_detection - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:55:49,908 - stpipe.Image3Pipeline.outlier_detection - INFO - Exposure jw02666001001_02101_00004_mirimage_outlier_i2d.fits saved to file\n",
      "2022-12-14 12:55:53,680 - stpipe.Image3Pipeline.outlier_detection - INFO - Blotting median...\n",
      "2022-12-14 12:55:54,316 - stpipe.Image3Pipeline.outlier_detection - INFO - Blotting (1024, 1032) <-- (1172, 1178)\n",
      "2022-12-14 12:55:55,306 - stpipe.Image3Pipeline.outlier_detection - INFO - Blotting (1024, 1032) <-- (1172, 1178)\n",
      "2022-12-14 12:55:56,308 - stpipe.Image3Pipeline.outlier_detection - INFO - Blotting (1024, 1032) <-- (1172, 1178)\n",
      "2022-12-14 12:55:57,327 - stpipe.Image3Pipeline.outlier_detection - INFO - Blotting (1024, 1032) <-- (1172, 1178)\n",
      "2022-12-14 12:55:57,715 - stpipe.Image3Pipeline.outlier_detection - INFO - Flagging outliers\n",
      "2022-12-14 12:55:58,093 - stpipe.Image3Pipeline.outlier_detection - INFO - New pixels flagged as outliers: 1227 (0.12%)\n",
      "2022-12-14 12:55:58,328 - stpipe.Image3Pipeline.outlier_detection - INFO - New pixels flagged as outliers: 614 (0.06%)\n",
      "2022-12-14 12:55:58,573 - stpipe.Image3Pipeline.outlier_detection - INFO - New pixels flagged as outliers: 1091 (0.10%)\n",
      "2022-12-14 12:55:58,813 - stpipe.Image3Pipeline.outlier_detection - INFO - New pixels flagged as outliers: 1148 (0.11%)\n",
      "2022-12-14 12:55:59,181 - stpipe.Image3Pipeline.outlier_detection - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/jw02666001001_02101_00001_mirimage_a3001_crf.fits\n",
      "2022-12-14 12:55:59,536 - stpipe.Image3Pipeline.outlier_detection - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/jw02666001001_02101_00002_mirimage_a3001_crf.fits\n",
      "2022-12-14 12:55:59,889 - stpipe.Image3Pipeline.outlier_detection - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/jw02666001001_02101_00003_mirimage_a3001_crf.fits\n",
      "2022-12-14 12:56:00,235 - stpipe.Image3Pipeline.outlier_detection - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/jw02666001001_02101_00004_mirimage_a3001_crf.fits\n",
      "2022-12-14 12:56:00,236 - stpipe.Image3Pipeline.outlier_detection - INFO - Step outlier_detection done\n",
      "2022-12-14 12:56:00,350 - stpipe.Image3Pipeline.resample - INFO - Step resample running with args (<ModelContainer>,).\n",
      "2022-12-14 12:56:00,352 - stpipe.Image3Pipeline.resample - INFO - Step resample parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'i2d', 'search_output_file': True, 'input_dir': './test_mosaic/F560W_level3_gaia', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}\n",
      "2022-12-14 12:56:00,395 - stpipe.Image3Pipeline.resample - INFO - Using drizpars reference file: /grp/crds/cache/references/jwst/jwst_miri_drizpars_0001.fits\n",
      "2022-12-14 12:56:00,483 - stpipe.Image3Pipeline.resample - INFO - Driz parameter kernel: square\n",
      "2022-12-14 12:56:00,484 - stpipe.Image3Pipeline.resample - INFO - Driz parameter pixfrac: 1.0\n",
      "2022-12-14 12:56:00,484 - stpipe.Image3Pipeline.resample - INFO - Driz parameter fillval: INDEF\n",
      "2022-12-14 12:56:00,485 - stpipe.Image3Pipeline.resample - INFO - Driz parameter weight_type: ivm\n",
      "2022-12-14 12:56:00,485 - stpipe.Image3Pipeline.resample - INFO - Output pixel scale ratio: 1.0\n",
      "2022-12-14 12:56:00,679 - stpipe.Image3Pipeline.resample - INFO - Blending metadata for F560W\n",
      "2022-12-14 12:56:01,630 - stpipe.Image3Pipeline.resample - INFO - Resampling science data\n",
      "2022-12-14 12:56:02,167 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:02,834 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:03,510 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:04,197 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:04,345 - stpipe.Image3Pipeline.resample - INFO - Resampling var_rnoise\n",
      "2022-12-14 12:56:04,886 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:05,604 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:06,287 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:06,985 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:07,159 - stpipe.Image3Pipeline.resample - INFO - Resampling var_poisson\n",
      "2022-12-14 12:56:07,691 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:08,390 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:09,121 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:09,817 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:09,979 - stpipe.Image3Pipeline.resample - INFO - Resampling var_flat\n",
      "2022-12-14 12:56:10,519 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:11,216 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:11,919 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:12,627 - stpipe.Image3Pipeline.resample - INFO - Drizzling (1024, 1032) --> (1172, 1178)\n",
      "2022-12-14 12:56:12,828 - stpipe.Image3Pipeline.resample - INFO - Update S_REGION to POLYGON ICRS  308.798651256 60.116531507 308.852064085 60.092137428 308.901300681 60.118876807 308.847880946 60.143290713\n",
      "2022-12-14 12:56:13,508 - stpipe.Image3Pipeline.resample - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_i2d.fits\n",
      "2022-12-14 12:56:13,509 - stpipe.Image3Pipeline.resample - INFO - Step resample done\n",
      "2022-12-14 12:56:13,633 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog running with args (<ImageModel(1172, 1178) from F560W_i2d.fits>,).\n",
      "2022-12-14 12:56:13,635 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': './test_mosaic/F560W_level3_gaia', 'bkg_boxsize': 1000, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 25, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:56:13,678 - stpipe.Image3Pipeline.source_catalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:56:13,710 - stpipe.Image3Pipeline.source_catalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:56:13,712 - stpipe.Image3Pipeline.source_catalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:56:13,713 - stpipe.Image3Pipeline.source_catalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:56:13,714 - stpipe.Image3Pipeline.source_catalog - INFO - Filter: F560W\n",
      "2022-12-14 12:56:13,714 - stpipe.Image3Pipeline.source_catalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:56:13,935 - stpipe.Image3Pipeline.source_catalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:56:14,029 - stpipe.Image3Pipeline.source_catalog - INFO - Background could not be estimated in meshes. Using the entire unmasked array for background estimation: bkg_boxsize=(1172, 1178).\n",
      "2022-12-14 12:56:14,210 - stpipe.Image3Pipeline.source_catalog - INFO - Detected 165 sources\n",
      "2022-12-14 12:56:14,548 - stpipe.Image3Pipeline.source_catalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_cat.ecsv\n",
      "2022-12-14 12:56:14,729 - stpipe.Image3Pipeline.source_catalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_segm.fits\n",
      "2022-12-14 12:56:14,731 - stpipe.Image3Pipeline.source_catalog - INFO - Wrote segmentation map: F560W_segm.fits\n",
      "2022-12-14 12:56:14,738 - stpipe.Image3Pipeline.source_catalog - INFO - Step source_catalog done\n",
      "2022-12-14 12:56:14,740 - stpipe.Image3Pipeline - INFO - Step Image3Pipeline done\n"
     ]
    }
   ],
   "source": [
    "image3.run(miri_asn_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "226ad48e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:40,262 - stpipe - WARNING - <>:15: DeprecationWarning: invalid escape sequence '\\.'\n",
      "\n",
      "2022-12-14 12:58:40,265 - stpipe - WARNING - <>:15: DeprecationWarning: invalid escape sequence '\\.'\n",
      "\n",
      "2022-12-14 12:58:40,266 - stpipe - WARNING - /var/folders/16/91kljsb17fd5xnw3fr_fdj70000149/T/ipykernel_81315/3323345972.py:15: DeprecationWarning: invalid escape sequence '\\.'\n",
      "  outfile = re.sub('_([a-zA-Z0-9]+)\\.fits$','',file)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running ./test_mosaic/F560W_level3_gaia/F560W_snr5_npix15_cat.ecsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:40,712 - stpipe - INFO - PARS-SOURCECATALOGSTEP parameters found: /grp/crds/cache/references/jwst/jwst_miri_pars-sourcecatalogstep_0020.asdf\n",
      "2022-12-14 12:58:40,796 - stpipe.sourcecatalog - INFO - SourceCatalogStep instance created.\n",
      "2022-12-14 12:58:40,933 - stpipe - WARNING - /Users/arest/anaconda3/envs/jwst/lib/python3.10/site-packages/stpipe/step.py:419: ResourceWarning: unclosed file <_io.BufferedReader name='./test_mosaic/F560W_level3_gaia/F560W_i2d.fits'>\n",
      "  gc.collect()\n",
      "\n",
      "2022-12-14 12:58:40,945 - stpipe.sourcecatalog - INFO - Step sourcecatalog running with args ('./test_mosaic/F560W_level3_gaia/F560W_i2d.fits',).\n",
      "2022-12-14 12:58:40,948 - stpipe.sourcecatalog - INFO - Step sourcecatalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/Users/arest/nircam/jhat/test_mosaic/F560W_level3_gaia/F560W_snr5_npix15_cat.ecsv', 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 5.0, 'npixels': 15, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n",
      "2022-12-14 12:58:41,281 - stpipe.sourcecatalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:58:41,365 - stpipe.sourcecatalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:58:41,366 - stpipe.sourcecatalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:58:41,367 - stpipe.sourcecatalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:58:41,367 - stpipe.sourcecatalog - INFO - Filter: F560W\n",
      "2022-12-14 12:58:41,368 - stpipe.sourcecatalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:58:41,510 - stpipe.sourcecatalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:58:41,984 - stpipe.sourcecatalog - INFO - Detected 121 sources\n",
      "2022-12-14 12:58:42,239 - stpipe.sourcecatalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_snr5_npix15_cat.ecsv\n",
      "2022-12-14 12:58:42,403 - stpipe.sourcecatalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_snr5_npix15_segm.fits\n",
      "2022-12-14 12:58:42,404 - stpipe.sourcecatalog - INFO - Wrote segmentation map: F560W_snr5_npix15_segm.fits\n",
      "2022-12-14 12:58:42,412 - stpipe.sourcecatalog - INFO - Step sourcecatalog done\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running ./test_mosaic/F560W_level3_gaia/F560W_snr5_npix10_cat.ecsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:42,666 - stpipe - INFO - PARS-SOURCECATALOGSTEP parameters found: /grp/crds/cache/references/jwst/jwst_miri_pars-sourcecatalogstep_0020.asdf\n",
      "2022-12-14 12:58:42,696 - stpipe.sourcecatalog - INFO - SourceCatalogStep instance created.\n",
      "2022-12-14 12:58:42,802 - stpipe - WARNING - /Users/arest/anaconda3/envs/jwst/lib/python3.10/site-packages/stpipe/step.py:419: ResourceWarning: unclosed file <_io.BufferedReader name='./test_mosaic/F560W_level3_gaia/F560W_i2d.fits'>\n",
      "  gc.collect()\n",
      "\n",
      "2022-12-14 12:58:42,812 - stpipe.sourcecatalog - INFO - Step sourcecatalog running with args ('./test_mosaic/F560W_level3_gaia/F560W_i2d.fits',).\n",
      "2022-12-14 12:58:42,815 - stpipe.sourcecatalog - INFO - Step sourcecatalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/Users/arest/nircam/jhat/test_mosaic/F560W_level3_gaia/F560W_snr5_npix10_cat.ecsv', 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 5.0, 'npixels': 10, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n",
      "2022-12-14 12:58:43,132 - stpipe.sourcecatalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:58:43,164 - stpipe.sourcecatalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:58:43,164 - stpipe.sourcecatalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:58:43,165 - stpipe.sourcecatalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:58:43,165 - stpipe.sourcecatalog - INFO - Filter: F560W\n",
      "2022-12-14 12:58:43,166 - stpipe.sourcecatalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:58:43,287 - stpipe.sourcecatalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:58:43,809 - stpipe.sourcecatalog - INFO - Detected 194 sources\n",
      "2022-12-14 12:58:44,192 - stpipe.sourcecatalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_snr5_npix10_cat.ecsv\n",
      "2022-12-14 12:58:44,374 - stpipe.sourcecatalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_snr5_npix10_segm.fits\n",
      "2022-12-14 12:58:44,376 - stpipe.sourcecatalog - INFO - Wrote segmentation map: F560W_snr5_npix10_segm.fits\n",
      "2022-12-14 12:58:44,384 - stpipe.sourcecatalog - INFO - Step sourcecatalog done\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running ./test_mosaic/F560W_level3_gaia/F560W_snr3_npix15_cat.ecsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:44,620 - stpipe - INFO - PARS-SOURCECATALOGSTEP parameters found: /grp/crds/cache/references/jwst/jwst_miri_pars-sourcecatalogstep_0020.asdf\n",
      "2022-12-14 12:58:44,653 - stpipe.sourcecatalog - INFO - SourceCatalogStep instance created.\n",
      "2022-12-14 12:58:44,787 - stpipe.sourcecatalog - INFO - Step sourcecatalog running with args ('./test_mosaic/F560W_level3_gaia/F560W_i2d.fits',).\n",
      "2022-12-14 12:58:44,789 - stpipe.sourcecatalog - INFO - Step sourcecatalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/Users/arest/nircam/jhat/test_mosaic/F560W_level3_gaia/F560W_snr3_npix15_cat.ecsv', 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 15, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n",
      "2022-12-14 12:58:45,080 - stpipe.sourcecatalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:58:45,112 - stpipe.sourcecatalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:58:45,113 - stpipe.sourcecatalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:58:45,114 - stpipe.sourcecatalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:58:45,115 - stpipe.sourcecatalog - INFO - Filter: F560W\n",
      "2022-12-14 12:58:45,115 - stpipe.sourcecatalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:58:45,256 - stpipe.sourcecatalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:58:45,707 - stpipe.sourcecatalog - INFO - Detected 238 sources\n",
      "2022-12-14 12:58:46,044 - stpipe.sourcecatalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_snr3_npix15_cat.ecsv\n",
      "2022-12-14 12:58:46,220 - stpipe.sourcecatalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_snr3_npix15_segm.fits\n",
      "2022-12-14 12:58:46,221 - stpipe.sourcecatalog - INFO - Wrote segmentation map: F560W_snr3_npix15_segm.fits\n",
      "2022-12-14 12:58:46,228 - stpipe.sourcecatalog - INFO - Step sourcecatalog done\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running ./test_mosaic/F560W_level3_gaia/F560W_snr3_npix10_cat.ecsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:46,465 - stpipe - INFO - PARS-SOURCECATALOGSTEP parameters found: /grp/crds/cache/references/jwst/jwst_miri_pars-sourcecatalogstep_0020.asdf\n",
      "2022-12-14 12:58:46,498 - stpipe.sourcecatalog - INFO - SourceCatalogStep instance created.\n",
      "2022-12-14 12:58:46,644 - stpipe.sourcecatalog - INFO - Step sourcecatalog running with args ('./test_mosaic/F560W_level3_gaia/F560W_i2d.fits',).\n",
      "2022-12-14 12:58:46,646 - stpipe.sourcecatalog - INFO - Step sourcecatalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/Users/arest/nircam/jhat/test_mosaic/F560W_level3_gaia/F560W_snr3_npix10_cat.ecsv', 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 3.0, 'npixels': 10, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n",
      "2022-12-14 12:58:46,912 - stpipe.sourcecatalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:58:46,962 - stpipe.sourcecatalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:58:46,964 - stpipe.sourcecatalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:58:46,965 - stpipe.sourcecatalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:58:46,966 - stpipe.sourcecatalog - INFO - Filter: F560W\n",
      "2022-12-14 12:58:46,966 - stpipe.sourcecatalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:58:47,107 - stpipe.sourcecatalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:58:47,591 - stpipe.sourcecatalog - INFO - Detected 352 sources\n",
      "2022-12-14 12:58:48,099 - stpipe.sourcecatalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_snr3_npix10_cat.ecsv\n",
      "2022-12-14 12:58:48,290 - stpipe.sourcecatalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_snr3_npix10_segm.fits\n",
      "2022-12-14 12:58:48,292 - stpipe.sourcecatalog - INFO - Wrote segmentation map: F560W_snr3_npix10_segm.fits\n",
      "2022-12-14 12:58:48,299 - stpipe.sourcecatalog - INFO - Step sourcecatalog done\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running ./test_mosaic/F560W_level3_gaia/F560W_snr2_npix15_cat.ecsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:48,563 - stpipe - INFO - PARS-SOURCECATALOGSTEP parameters found: /grp/crds/cache/references/jwst/jwst_miri_pars-sourcecatalogstep_0020.asdf\n",
      "2022-12-14 12:58:48,592 - stpipe.sourcecatalog - INFO - SourceCatalogStep instance created.\n",
      "2022-12-14 12:58:48,728 - stpipe.sourcecatalog - INFO - Step sourcecatalog running with args ('./test_mosaic/F560W_level3_gaia/F560W_i2d.fits',).\n",
      "2022-12-14 12:58:48,730 - stpipe.sourcecatalog - INFO - Step sourcecatalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/Users/arest/nircam/jhat/test_mosaic/F560W_level3_gaia/F560W_snr2_npix15_cat.ecsv', 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 2.0, 'npixels': 15, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n",
      "2022-12-14 12:58:49,032 - stpipe.sourcecatalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:58:49,080 - stpipe.sourcecatalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:58:49,081 - stpipe.sourcecatalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:58:49,081 - stpipe.sourcecatalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:58:49,082 - stpipe.sourcecatalog - INFO - Filter: F560W\n",
      "2022-12-14 12:58:49,083 - stpipe.sourcecatalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:58:49,215 - stpipe.sourcecatalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:58:49,693 - stpipe.sourcecatalog - INFO - Detected 495 sources\n",
      "2022-12-14 12:58:50,300 - stpipe.sourcecatalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_snr2_npix15_cat.ecsv\n",
      "2022-12-14 12:58:50,472 - stpipe.sourcecatalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_snr2_npix15_segm.fits\n",
      "2022-12-14 12:58:50,474 - stpipe.sourcecatalog - INFO - Wrote segmentation map: F560W_snr2_npix15_segm.fits\n",
      "2022-12-14 12:58:50,481 - stpipe.sourcecatalog - INFO - Step sourcecatalog done\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running ./test_mosaic/F560W_level3_gaia/F560W_snr2_npix10_cat.ecsv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-14 12:58:50,727 - stpipe - INFO - PARS-SOURCECATALOGSTEP parameters found: /grp/crds/cache/references/jwst/jwst_miri_pars-sourcecatalogstep_0020.asdf\n",
      "2022-12-14 12:58:50,777 - stpipe.sourcecatalog - INFO - SourceCatalogStep instance created.\n",
      "2022-12-14 12:58:50,899 - stpipe - WARNING - /Users/arest/anaconda3/envs/jwst/lib/python3.10/site-packages/stpipe/step.py:419: ResourceWarning: unclosed file <_io.BufferedReader name='./test_mosaic/F560W_level3_gaia/F560W_i2d.fits'>\n",
      "  gc.collect()\n",
      "\n",
      "2022-12-14 12:58:50,910 - stpipe.sourcecatalog - INFO - Step sourcecatalog running with args ('./test_mosaic/F560W_level3_gaia/F560W_i2d.fits',).\n",
      "2022-12-14 12:58:50,913 - stpipe.sourcecatalog - INFO - Step sourcecatalog parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': '/Users/arest/nircam/jhat/test_mosaic/F560W_level3_gaia/F560W_snr2_npix10_cat.ecsv', 'output_dir': './test_mosaic/F560W_level3_gaia', 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': True, 'skip': False, 'suffix': 'cat', 'search_output_file': True, 'input_dir': '', 'bkg_boxsize': 100, 'kernel_fwhm': 2.0, 'snr_threshold': 2.0, 'npixels': 10, 'deblend': False, 'aperture_ee1': 30, 'aperture_ee2': 50, 'aperture_ee3': 70, 'ci1_star_threshold': 2.0, 'ci2_star_threshold': 1.8}\n",
      "2022-12-14 12:58:51,188 - stpipe.sourcecatalog - INFO - Using APCORR reference file: /grp/crds/cache/references/jwst/jwst_miri_apcorr_0008.fits\n",
      "2022-12-14 12:58:51,228 - stpipe.sourcecatalog - INFO - Using ABVEGAOFFSET reference file: /grp/crds/cache/references/jwst/jwst_miri_abvegaoffset_0001.asdf\n",
      "2022-12-14 12:58:51,229 - stpipe.sourcecatalog - INFO - Instrument: MIRI\n",
      "2022-12-14 12:58:51,230 - stpipe.sourcecatalog - INFO - Detector: MIRIMAGE\n",
      "2022-12-14 12:58:51,230 - stpipe.sourcecatalog - INFO - Filter: F560W\n",
      "2022-12-14 12:58:51,231 - stpipe.sourcecatalog - INFO - Subarray: FULL\n",
      "2022-12-14 12:58:51,364 - stpipe.sourcecatalog - INFO - AB to Vega magnitude offset 3.76066\n",
      "2022-12-14 12:58:51,844 - stpipe.sourcecatalog - INFO - Detected 725 sources\n",
      "2022-12-14 12:58:52,683 - stpipe.sourcecatalog - INFO - Wrote source catalog: ./test_mosaic/F560W_level3_gaia/F560W_snr2_npix10_cat.ecsv\n",
      "2022-12-14 12:58:52,857 - stpipe.sourcecatalog - INFO - Saved model in ./test_mosaic/F560W_level3_gaia/F560W_snr2_npix10_segm.fits\n",
      "2022-12-14 12:58:52,858 - stpipe.sourcecatalog - INFO - Wrote segmentation map: F560W_snr2_npix10_segm.fits\n",
      "2022-12-14 12:58:52,865 - stpipe.sourcecatalog - INFO - Step sourcecatalog done\n"
     ]
    }
   ],
   "source": [
    "from jwst.source_catalog import SourceCatalogStep\n",
    "import re\n",
    "\n",
    "snr_thresholds = [5.0,3.0,2.0]\n",
    "#snr_thresholds = [2.0]\n",
    "npixel_vals = [15,10]\n",
    "\n",
    "\n",
    "\n",
    "files = glob.glob(f'{outdir_level3}/{filter_name}_i2d.fits')\n",
    "\n",
    "for file in files:\n",
    "    for snr in snr_thresholds:\n",
    "        for npix in npixel_vals:\n",
    "            outfile = re.sub('_([a-zA-Z0-9]+)\\.fits$','',file)\n",
    "            outfile += f'_snr{snr:.0f}_npix{npix:.0f}_cat.ecsv'\n",
    "            print(f'Running {outfile}')\n",
    "            result = SourceCatalogStep.call(file, \n",
    "                                            snr_threshold=float(snr),\n",
    "                                            npixels = int(npix), bkg_boxsize = 100,\n",
    "                                            output_file=outfile, \n",
    "                                            save_results=True,output_dir=f'{outdir_level3}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "214be3cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}