Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[hail] add contig_recoding to import_bed and import_locus_intervals #7013

Merged
merged 9 commits into from
Sep 9, 2019
Merged

[hail] add contig_recoding to import_bed and import_locus_intervals #7013

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5f82d1c
[hail] add contig_recoding to import_bed and import_locus_intervals
Sep 6, 2019
53e5d08
fix
Sep 7, 2019
8fb9664
fix
Sep 7, 2019
60a50b1
address comments
Sep 7, 2019
ba4aa70
simplify
Sep 7, 2019
a79f9c7
fix test
Sep 9, 2019
6ca3ae4
banish assertTrue(... == ...)
Sep 9, 2019
18b72e6
more comments addressed
Sep 9, 2019
a5b81ce
fix
Sep 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 42 additions & 8 deletions hail/python/hail/methods/impex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,8 +516,13 @@ def export_vcf(dataset, output, append_to_header=None, parallel=None, metadata=N
@typecheck(path=str,
reference_genome=nullable(reference_genome_type),
skip_invalid_intervals=bool,
contig_recoding=nullable(dictof(str, str)),
kwargs=anytype)
def import_locus_intervals(path, reference_genome='default', skip_invalid_intervals=False, **kwargs) -> Table:
def import_locus_intervals(path,
reference_genome='default',
skip_invalid_intervals=False,
contig_recoding=None,
**kwargs) -> Table:
"""Import a locus interval list as a :class:`.Table`.

Examples
Expand Down Expand Up @@ -576,6 +581,10 @@ def import_locus_intervals(path, reference_genome='default', skip_invalid_interv
skip_invalid_intervals : :obj:`bool`
If ``True`` and `reference_genome` is not ``None``, skip lines with
intervals that are not consistent with the reference genome.
contig_recoding: :obj:`dict` of (:obj:`str`, :obj:`str`)
Mapping from contig name in file to contig name in loaded dataset.
All contigs must be present in the `reference_genome`, so this is
useful for mapping differently-formatted data onto known references.
**kwargs
Additional optional arguments to :func:`import_table` are valid
arguments here except: `no_header`, `comment`, `impute`, and
Expand All @@ -587,6 +596,14 @@ def import_locus_intervals(path, reference_genome='default', skip_invalid_interv
Interval-keyed table.
"""

if contig_recoding is not None:
contig_recoding = hl.literal(contig_recoding)

def recode_contig(x):
if contig_recoding is None:
return x
return contig_recoding.get(x, x)

t = import_table(path, comment="@", impute=False, no_header=True,
types={'f0': tstr, 'f1': tint32, 'f2': tint32,
'f3': tstr, 'f4': tstr},
Expand All @@ -601,7 +618,7 @@ def import_locus_intervals(path, reference_genome='default', skip_invalid_interv

def checked_match_interval_expr(match):
return hl.or_missing(hl.len(match) == 3,
locus_interval_expr(match[0],
locus_interval_expr(recode_contig(match[0]),
hl.int32(match[1]),
hl.int32(match[2]),
True,
Expand All @@ -613,7 +630,7 @@ def checked_match_interval_expr(match):
hl.bind(t['f0'].first_match_in(interval_regex),
lambda match: hl.cond(hl.bool(skip_invalid_intervals),
checked_match_interval_expr(match),
locus_interval_expr(match[0],
locus_interval_expr(recode_contig(match[0]),
hl.int32(match[1]),
hl.int32(match[2]),
True,
Expand All @@ -624,7 +641,7 @@ def checked_match_interval_expr(match):
t = t.select(interval=expr)

elif t.row.dtype == tstruct(f0=tstr, f1=tint32, f2=tint32):
t = t.select(interval=locus_interval_expr(t['f0'],
t = t.select(interval=locus_interval_expr(recode_contig(t['f0']),
t['f1'],
t['f2'],
True,
Expand All @@ -633,7 +650,7 @@ def checked_match_interval_expr(match):
skip_invalid_intervals))

elif t.row.dtype == tstruct(f0=tstr, f1=tint32, f2=tint32, f3=tstr, f4=tstr):
t = t.select(interval=locus_interval_expr(t['f0'],
t = t.select(interval=locus_interval_expr(recode_contig(t['f0']),
t['f1'],
t['f2'],
True,
Expand All @@ -657,8 +674,13 @@ def checked_match_interval_expr(match):
@typecheck(path=str,
reference_genome=nullable(reference_genome_type),
skip_invalid_intervals=bool,
contig_recoding=nullable(dictof(str, str)),
kwargs=anytype)
def import_bed(path, reference_genome='default', skip_invalid_intervals=False, **kwargs) -> Table:
def import_bed(path,
reference_genome='default',
skip_invalid_intervals=False,
contig_recoding=None,
**kwargs) -> Table:
"""Import a UCSC BED file as a :class:`.Table`.

Examples
Expand Down Expand Up @@ -731,6 +753,10 @@ def import_bed(path, reference_genome='default', skip_invalid_intervals=False, *
skip_invalid_intervals : :obj:`bool`
If ``True`` and `reference_genome` is not ``None``, skip lines with
intervals that are not consistent with the reference genome.
contig_recoding: :obj:`dict` of (:obj:`str`, :obj:`str`)
Mapping from contig name in BED to contig name in loaded dataset.
All contigs must be present in the `reference_genome`, so this is
useful for mapping differently-formatted data onto known references.
**kwargs
Additional optional arguments to :func:`import_table` are valid arguments here except:
`no_header`, `delimiter`, `impute`, `skip_blank_lines`, `types`, and `comment` as these
Expand All @@ -752,8 +778,16 @@ def import_bed(path, reference_genome='default', skip_invalid_intervals=False, *
r"""^\w+=("[\w\d ]+"|\d+).*"""],
**kwargs)

if contig_recoding is not None:
contig_recoding = hl.literal(contig_recoding)

def recode_contig(x):
if contig_recoding is None:
return x
return contig_recoding.get(x, x)

if t.row.dtype == tstruct(f0=tstr, f1=tint32, f2=tint32):
t = t.select(interval=locus_interval_expr(t['f0'],
t = t.select(interval=locus_interval_expr(recode_contig(t['f0']),
t['f1'] + 1,
t['f2'] + 1,
True,
Expand All @@ -762,7 +796,7 @@ def import_bed(path, reference_genome='default', skip_invalid_intervals=False, *
skip_invalid_intervals))

elif len(t.row) >= 4 and tstruct(**dict([(n, typ) for n, typ in t.row.dtype._field_types.items()][:4])) == tstruct(f0=tstr, f1=tint32, f2=tint32, f3=tstr):
t = t.select(interval=locus_interval_expr(t['f0'],
t = t.select(interval=locus_interval_expr(recode_contig(t['f0']),
t['f1'] + 1,
t['f2'] + 1,
True,
Expand Down
17 changes: 17 additions & 0 deletions hail/python/test/hail/methods/test_impex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1223,6 +1223,15 @@ def test_import_locus_intervals_no_reference_specified(self):
self.assertTrue(t.count() == 2)
self.assertEqual(t.interval.dtype.point_type, hl.tstruct(contig=hl.tstr, position=hl.tint32))

def test_import_locus_intervals_recoding(self):
interval_file = resource('annotinterall.grch38.no.chr.interval_list')
t = hl.import_locus_intervals(interval_file,
contig_recoding={str(i): f'chr{i}' for i in [*range(1, 23), 'X', 'Y', 'M']},
reference_genome='GRCh38')
t._force_count()
self.assertTrue(t.count() == 3)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can combine these two lines as you did below.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also, don't use assertTrue. use assertEqual, but standard python assert t.count() == 3 is preferred with pytest

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed every occurrence of assertTrue(... == ...) from this file.

self.assertEqual(t.interval.dtype.point_type, hl.tlocus('GRCh38'))

def test_import_locus_intervals_badly_defined_intervals(self):
interval_file = resource('example3.interval_list')
t = hl.import_locus_intervals(interval_file, reference_genome='GRCh37', skip_invalid_intervals=True)
Expand Down Expand Up @@ -1263,6 +1272,14 @@ def test_import_bed(self):

self.assertEqual(t.interval.collect(), hl.eval(expected))

def test_import_bed_recoding(self):
bed_file = resource('some-missing-chr-grch38.bed')
bed = hl.import_bed(bed_file,
reference_genome='GRCh38',
contig_recoding={str(i): f'chr{i}' for i in [*range(1, 23), 'X', 'Y', 'M']})
self.assertEqual(bed._force_count(), 3)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this be 5?

self.assertEqual(bed.interval.dtype.point_type, hl.tlocus('GRCh38'))

def test_import_bed_no_reference_specified(self):
bed_file = resource('example1.bed')
t = hl.import_bed(bed_file, reference_genome=None)
Expand Down
3 changes: 3 additions & 0 deletions hail/src/test/resources/annotinterall.grch38.no.chr.interval_list
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
22 17333902 17370919 + A
22 17348324 17348324 + B
chr22 17348326 17348330 + B
6 changes: 6 additions & 0 deletions hail/src/test/resources/some-missing-chr-grch38.bed
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
track name="BedTest"
20 2 10 gene0
20 1 14000000 gene1
20 17000000 18000000 gene2
chr20 63025510 63025520 gene3
chr20 5 5 gene4