Skip to content

Commit

Permalink
turn off poppy multiprocessing by default
Browse files Browse the repository at this point in the history
  • Loading branch information
@JarronL
JarronL committed May 18, 2020
1 parent 5416ac7 commit 1ad1403
Show file tree
Hide file tree
Showing 2 changed files with 250 additions and 22 deletions.
255 changes: 240 additions & 15 deletions dev_utils/WFE_coefficients.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1800,15 +1800,16 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"from pynrc.nrc_utils import gen_webbpsf_psf\n",
"from pynrc.nrc_utils import poppy\n",
"def perform_benchmarks(filter='F430M', pupil=None, mask=None, module='A',\n",
" fov_pix=33, oversample=4, include_si_wfe=True, \n",
" use_legendre=True, force=True, save=True, \n",
" do_webbpsf=True, **kwargs):\n",
" use_legendre=True, force=False, save=True, \n",
" do_webbpsf=True, use_mp=None, nproc=None, **kwargs):\n",
" \n",
" import datetime, time\n",
" \n",
Expand All @@ -1833,33 +1834,72 @@
" kwargs['save_name'] = None\n",
" \n",
" pynrc.setup_logging('WARN', verbose=False)\n",
" \n",
" tdict = {\n",
" 'webbpsf': None,\n",
" 'pynrc_coeff': None,\n",
" 'pynrc_drift': None,\n",
" 'pynrc_field': None,\n",
" 'pynrc_psf': None,\n",
" }\n",
" \n",
" # Multiprocessing cases\n",
" use_mp_def = poppy.conf.use_multiprocessing\n",
" nproc_def = poppy.conf.n_processes\n",
" if use_mp is not None:\n",
" poppy.conf.use_multiprocessing = use_mp\n",
" if nproc is not None:\n",
" poppy.conf.n_processes = nproc\n",
"\n",
" ####\n",
" # WebbPSF PSF generation\n",
" ####\n",
" if do_webbpsf:\n",
" t0 = time.time()\n",
" hdul = gen_webbpsf_psf(filter, **kwargs)\n",
" t1 = time.time()\n",
" time_string = 'Took {:.2f} seconds to generate WebbPSF PSF'.format(t1-t0)\n",
" dt = t1-t0\n",
" \n",
" tdict['webbpsf'] = dt\n",
" time_string = 'Took {:.2f} seconds to generate WebbPSF PSF'.format(dt)\n",
" print(time_string) \n",
" \n",
" ####\n",
" # pyNRC coefficients\n",
" ####\n",
" t0 = time.time()\n",
" nrc = pynrc.NIRCam(filter=filter, **kwargs)\n",
" t1 = time.time()\n",
" time_string = 'Took {:.2f} seconds to generate pyNRC coefficients'.format(t1-t0)\n",
"\n",
" tdict['pynrc_coeff'] = dt = t1-t0\n",
" time_string = 'Took {:.2f} seconds to generate pyNRC coefficients'.format(dt)\n",
" print(time_string)\n",
"\n",
" ####\n",
" # WFE drift coefficient generation\n",
" ####\n",
" t0 = time.time()\n",
" nrc.wfe_drift = True\n",
" t1 = time.time()\n",
" time_string = 'Took {:.2f} seconds to generate WFE Drift coefficients'.format(t1-t0)\n",
" dt = t1-t0\n",
"\n",
" tdict['pynrc_drift'] = dt = t1-t0\n",
" time_string = 'Took {:.2f} seconds to generate WFE Drift coefficients'.format(dt)\n",
" print(time_string)\n",
"\n",
" ####\n",
" # Field coefficient generation\n",
" ####\n",
" t0 = time.time()\n",
" nrc.wfe_field = True\n",
" t1 = time.time()\n",
" time_string = 'Took {:.2f} seconds to generate WFE Field coefficients'.format(t1-t0)\n",
"\n",
" tdict['pynrc_field'] = dt = t1-t0\n",
" time_string = 'Took {:.2f} seconds to generate WFE Field coefficients'.format(dt)\n",
" print(time_string)\n",
" \n",
" # Add aperture\n",
" # Add aperture manually to save time\n",
" # This doesn't really matter, just needs to be something plausible\n",
" apname = nrc.get_siaf_apname()\n",
" nrc._siaf_ap = nrc.siaf_nrc[apname]\n",
"\n",
Expand All @@ -1870,31 +1910,137 @@
" return_oversample=True)\n",
" t1 = time.time()\n",
" tarr.append(t1-t0)\n",
" time_string = 'Took {:.2f} seconds to generate pynrc PSF'.format(np.mean(tarr))\n",
" tdict['pynrc_psf'] = dt = np.mean(tarr)\n",
" time_string = 'Took {:.2f} seconds to generate pynrc PSF'.format(dt)\n",
" print(time_string)\n",
" \n",
"# return nrc\n",
" # Return defaults\n",
" poppy.conf.use_multiprocessing = use_mp_def\n",
" poppy.conf.n_processes = nproc_def\n",
" \n",
" return tdict\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 31,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Took 3.03 seconds to generate pyNRC coefficients\n",
"Took 5.11 seconds to generate WFE Drift coefficients\n",
"Took 9.37 seconds to generate WFE Field coefficients\n",
"Took 17.43 seconds to generate WebbPSF PSF\n",
"Took 2.63 seconds to generate pyNRC coefficients\n",
"Took 5.54 seconds to generate WFE Drift coefficients\n",
"Took 9.84 seconds to generate WFE Field coefficients\n",
"Took 0.90 seconds to generate pynrc PSF\n"
]
}
],
"source": [
"tdict = perform_benchmarks(fov_pix=129, do_webbpsf=True, use_mp=True)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Took 6.22 seconds to generate WebbPSF PSF\n",
"Took 3.17 seconds to generate pyNRC coefficients\n",
"Took 4.53 seconds to generate WFE Drift coefficients\n",
"Took 9.34 seconds to generate WFE Field coefficients\n",
"Took 1.21 seconds to generate pynrc PSF\n"
]
}
],
"source": [
"tdict = perform_benchmarks(fov_pix=129, do_webbpsf=True, use_mp=False)"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Took 18.11 seconds to generate WebbPSF PSF\n",
"Took 3.09 seconds to generate pyNRC coefficients\n",
"Took 4.81 seconds to generate WFE Drift coefficients\n",
"Took 7.49 seconds to generate WFE Field coefficients\n",
"Took 0.07 seconds to generate pynrc PSF\n"
]
}
],
"source": [
"perform_benchmarks(fov_pix=33, force=False, do_webbpsf=False)"
"tdict = perform_benchmarks(fov_pix=33, force=False, do_webbpsf=True, use_mp=True)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Took 23.42 seconds to generate WebbPSF PSF\n",
"Took 2.73 seconds to generate pyNRC coefficients\n",
"Took 4.38 seconds to generate WFE Drift coefficients\n",
"Took 6.57 seconds to generate WFE Field coefficients\n",
"Took 0.06 seconds to generate pynrc PSF\n"
]
}
],
"source": [
"tdict = perform_benchmarks(fov_pix=33, force=False, do_webbpsf=True, nproc=1)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Took 17.85 seconds to generate WebbPSF PSF\n",
"Took 2.85 seconds to generate pyNRC coefficients\n",
"Took 4.63 seconds to generate WFE Drift coefficients\n",
"Took 7.94 seconds to generate WFE Field coefficients\n",
"Took 0.11 seconds to generate pynrc PSF\n"
]
},
{
"data": {
"text/plain": [
"{'webbpsf': 17.852298974990845,\n",
" 'pynrc_coeff': 2.846261739730835,\n",
" 'pynrc_drift': 4.6318581104278564,\n",
" 'pynrc_field': 7.942458152770996,\n",
" 'pynrc_psf': 0.10760819911956787}"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"tdict = perform_benchmarks(fov_pix=33, force=False, do_webbpsf=True)"
]
},
{
Expand All @@ -1917,6 +2063,85 @@
"res.siaf_ap"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 3.1 s, sys: 1.63 s, total: 4.73 s\n",
"Wall time: 15 s\n"
]
}
],
"source": [
"nrc = webbpsf.NIRCam()\n",
"%time psf = nrc.calc_psf(fov_pixels=33, oversample=4) "
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"3"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"webbpsf.webbpsf_core.poppy.conf.n_processes"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<function pynrc.nrc_utils.nproc_use(fov_pix, oversample, nwavelengths, coron=False)>"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pynrc.nrc_utils."
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"webbpsf.webbpsf_core.poppy.conf.use_multiprocessing"
]
},
{
"cell_type": "code",
"execution_count": 15,
Expand Down
17 changes: 10 additions & 7 deletions pynrc/nrc_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@
import multiprocessing as mp
import traceback

from astropy.io import fits
from astropy.io import ascii
from astropy.io import fits, ascii
from astropy.table import Table
from astropy.time import Time
from astropy import units
Expand Down Expand Up @@ -72,14 +71,17 @@
raise ImportError('WebbPSF is not installed. pyNRC depends on its inclusion.')
# Check that minimum required version meets requirements
if not on_rtd:
_webbpsf_version_min = (0,7,0)
_webbpsf_version_min = (0,9,0)
_ = webbpsf.utils.get_webbpsf_data_path(_webbpsf_version_min)

# Link to WebbPSF's instance of poppy
from webbpsf.webbpsf_core import poppy

# Set up some poppy and webbpsf defaults
poppy.conf.use_multiprocessing = True # Assume multiprocessing
# Turn off multiprocessing, which is faster now due to improved
# underlying vectorization of poppy and numpy. Swapping large
# amount of data between processes is now the bottleneck for mp (5/18/2020).
poppy.conf.use_multiprocessing = False
# Only use this if you have the FFTW C library installed
# In general, numpy fft is actually pretty fast now, so default use_fftw=False
# It also doesn't play well with multiprocessing
Expand Down Expand Up @@ -1121,9 +1123,6 @@ def gen_psf_coeff(filter_or_bp, pupil=None, mask=None, module='A',
npsf = ndeg+1 if npsf<=ndeg else int(npsf)
waves = np.linspace(w1, w2, npsf)

# How many processors to split into?
nproc = nproc_use(fov_pix, oversample, npsf) if poppy.conf.use_multiprocessing else 1
_log.debug('nprocessors: {}; npsf: {}'.format(nproc, npsf))
# Change log levels to WARNING for pyNRC, WebbPSF, and POPPY
setup_logging('WARN', verbose=False)

Expand Down Expand Up @@ -1152,6 +1151,10 @@ def gen_psf_coeff(filter_or_bp, pupil=None, mask=None, module='A',
return hdu_list


# How many processors to split into?
nproc = nproc_use(fov_pix, oversample, npsf) #if poppy.conf.use_multiprocessing else 1
_log.debug('nprocessors: {}; npsf: {}'.format(nproc, npsf))

t0 = time.time()
# Setup the multiprocessing pool and arguments to pass to each pool
worker_arguments = [(inst, wlen, fov_pix, oversample) for wlen in waves]
Expand Down

0 comments on commit 1ad1403

Please sign in to comment.