Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

condense to a single read per file type with handling for partial reads #393

Merged
merged 5 commits into from
Jul 11, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 5 additions & 4 deletions docs/tutorial.rst
Original file line number Diff line number Diff line change
Expand Up @@ -682,7 +682,7 @@ or read in the header followed by the metadata (both shown below)
>>> uv.read_uvfits(filename, read_data=False)

>>> uv.read_uvfits(filename, read_data=False, read_metadata=False)
>>> uv.read_uvfits_metadata(filename)
>>> uv.read_uvfits(filename, read_data=False)

>>> print(uv.time_array.size)
1360
Expand All @@ -692,14 +692,14 @@ or read in the header followed by the metadata (both shown below)

# If the data_array, flag_array or nsample_array are needed later, they can be
# read into the existing object:
>>> uv.read_uvfits_data(filename)
>>> uv.read_uvfits(filename)
>>> print(uv.data_array.shape)
(1360, 1, 64, 4)

c) Reading only parts of uvfits data
***************************************
The same options that are available for the select function can also be passed to
read_uvfits or read_uvfits_data to do the select on the read, saving memory and
read_uvfits to do the select on the read, saving memory and
time if only a portion of the data are needed.
::

Expand All @@ -712,13 +712,14 @@ time if only a portion of the data are needed.
(1360, 1, 32, 4)

# Reading in the header and metadata can help with specifying what data to read in
>>> uv = UVData()
>>> uv.read_uvfits(filename, read_data=False)
>>> unique_times = np.unique(uv.time_array)
>>> print(unique_times.shape)
(15,)

>>> times_to_keep = unique_times[[0, 2, 4]]
>>> uv.read_uvfits_data(filename, times=times_to_keep)
>>> uv.read_uvfits(filename, times=times_to_keep)
>>> print(uv.data_array.shape)
(179, 1, 64, 4)

Expand Down
36 changes: 21 additions & 15 deletions pyuvdata/miriad.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,37 +37,43 @@ def _pol_to_ind(self, pol):
"polarization_array".format(pol=pol))
return pol_ind

def read_miriad(self, filepath, correct_lat_lon=True, run_check=True,
check_extra=True, run_check_acceptability=True, phase_type=None,
antenna_nums=None, ant_str=None, bls=None,
polarizations=None, time_range=None):
def read_miriad(self, filepath, antenna_nums=None, ant_str=None, bls=None,
polarizations=None, time_range=None, read_data=True,
phase_type=None, correct_lat_lon=True, run_check=True,
check_extra=True, run_check_acceptability=True):
"""
Read in data from a miriad file.

Args:
filepath: The miriad file directory to read from.
correct_lat_lon: flag -- that only matters if altitude is missing --
to update the latitude and longitude from the known_telescopes list
run_check: Option to check for the existence and proper shapes of
parameters after reading in the file. Default is True.
check_extra: Option to check optional parameters as well as required
ones. Default is True.
run_check_acceptability: Option to check acceptable range of the values of
parameters after reading in the file. Default is True.
antenna_nums: The antennas numbers to only read into the object.
antenna_nums: The antennas numbers to read into the object.
bls: A list of antenna number tuples (e.g. [(0,1), (3,2)]) or a list of
baseline 3-tuples (e.g. [(0,1,'xx'), (2,3,'yy')]) specifying baselines
to keep in the object. For length-2 tuples, the ordering of the numbers
within the tuple does not matter. For length-3 tuples, the polarization
string is in the order of the two antennas. If length-3 tuples are provided,
the polarizations argument below must be None.
string is in the order of the two antennas. If length-3 tuples are
provided, the polarizations argument below must be None.
ant_str: A string containing information about what kinds of visibility data
to read-in. Can be 'auto', 'cross', 'all'. Cannot provide ant_str if
antenna_nums and/or bls is not None.
polarizations: List of polarization integers or strings to read-in.
Ex: ['xx', 'yy', ...]
time_range: len-2 list containing min and max range of times (Julian Date) to read-in.
Ex: [2458115.20, 2458115.40]
read_data: Read in the visibility and flag data. If set to false,
only the metadata will be read in. Results in an incompletely
defined object (check will not pass). Default True.
phase_type: Either 'drift' meaning zenith drift, 'phased' meaning
the data are phased to a single RA/Dec or None and it will be
guessed at based on the file. Default None.
correct_lat_lon: flag -- that only matters if altitude is missing --
to update the latitude and longitude from the known_telescopes list
run_check: Option to check for the existence and proper shapes of
parameters after reading in the file. Default is True.
check_extra: Option to check optional parameters as well as required
ones. Default is True.
run_check_acceptability: Option to check acceptable range of the values of
parameters after reading in the file. Default is True.
"""
if not os.path.exists(filepath):
raise IOError(filepath + ' not found')
Expand Down
6 changes: 3 additions & 3 deletions pyuvdata/tests/test_miriad.py
Original file line number Diff line number Diff line change
Expand Up @@ -697,7 +697,7 @@ def test_readWriteReadMiriad():

# try metadata only read
uv_in = UVData()
uv_in.read_miriad_metadata(testfile)
uv_in.read_miriad(testfile, read_data=False)
nt.assert_equal(uv_in.time_array, None)
nt.assert_equal(uv_in.data_array, None)
metadata = ['antenna_positions', 'antenna_names', 'antenna_positions', 'channel_width',
Expand All @@ -708,11 +708,11 @@ def test_readWriteReadMiriad():
# test exceptions
# multiple file read-in
uv_in = UVData()
nt.assert_raises(ValueError, uv_in.read_miriad_metadata, [testfile, testfile])
nt.assert_raises(ValueError, uv_in.read_miriad, [testfile, testfile], read_data=False)
# read-in when data already exists
uv_in = UVData()
uv_in.read_miriad(testfile)
nt.assert_raises(ValueError, uv_in.read_miriad_metadata, testfile)
nt.assert_raises(ValueError, uv_in.read_miriad, testfile, read_data=False)

# test load_telescope_coords w/ blank Miriad
uv_in = Miriad()
Expand Down
12 changes: 4 additions & 8 deletions pyuvdata/tests/test_uvdata.py
Original file line number Diff line number Diff line change
Expand Up @@ -938,10 +938,8 @@ def test_select_not_inplace():
def test_reorder_pols():
# Test function to fix polarization order
uv1 = UVData()
testfile = os.path.join(
DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits')
uvtest.checkWarnings(
uv1.read_uvfits, [testfile], message='Telescope EVLA is not')
testfile = os.path.join(DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits')
uvtest.checkWarnings(uv1.read_uvfits, [testfile], message='Telescope EVLA is not')
uv2 = copy.deepcopy(uv1)
# reorder uv2 manually
order = [1, 3, 2, 0]
Expand All @@ -953,16 +951,14 @@ def test_reorder_pols():
nt.assert_equal(uv1, uv2)

# Restore original order
uvtest.checkWarnings(
uv1.read_uvfits, [testfile], message='Telescope EVLA is not')
uvtest.checkWarnings(uv1.read_uvfits, [testfile], message='Telescope EVLA is not')
uv2.reorder_pols()
nt.assert_equal(uv1, uv2)


def test_add():
uv_full = UVData()
testfile = os.path.join(
DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits')
testfile = os.path.join(DATA_PATH, 'day2_TDEM0003_10s_norx_1src_1spw.uvfits')
uvtest.checkWarnings(uv_full.read_uvfits, [testfile],
message='Telescope EVLA is not')

Expand Down
18 changes: 8 additions & 10 deletions pyuvdata/tests/test_uvfits.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ def test_ReadNRAO():
nt.assert_equal(expected_extra_keywords.sort(),
list(UV2.extra_keywords.keys()).sort())
nt.assert_raises(ValueError, UV2.check)
UV2.read_uvfits_metadata(testfile)
UV2.read_uvfits(testfile, read_data=False)
nt.assert_raises(ValueError, UV2.check)
UV2.read_uvfits_data(testfile)
UV2.read_uvfits(testfile)
nt.assert_equal(UV, UV2)

# test reading in header data first, then metadata & data
Expand All @@ -46,7 +46,7 @@ def test_ReadNRAO():
nt.assert_equal(expected_extra_keywords.sort(),
list(UV2.extra_keywords.keys()).sort())
nt.assert_raises(ValueError, UV2.check)
UV2.read_uvfits_data(testfile)
UV2.read_uvfits(testfile)
nt.assert_equal(UV, UV2)

del(UV)
Expand Down Expand Up @@ -433,15 +433,13 @@ def test_multi_files():
uv1.history = uv_full.history
nt.assert_equal(uv1, uv_full)

# check raises error if read_data is False
# check raises error if read_data and read_metadata are False
nt.assert_raises(ValueError, uv1.read_uvfits, [testfile1, testfile2],
read_data=False)
read_data=False, read_metadata=False)

# check raises error for read_uvfits_metadata
nt.assert_raises(ValueError, uv1.read_uvfits_metadata, [testfile1, testfile2])

# check raises error for read_uvfits_data
nt.assert_raises(ValueError, uv1.read_uvfits_data, [testfile1, testfile2])
# check raises error if read_data is False and read_metadata is True
nt.assert_raises(ValueError, uv1.read_uvfits, [testfile1, testfile2],
read_data=False, read_metadata=True)


def test_readMSWriteUVFits_CASAHistory():
Expand Down