Skip to content

Commit

Permalink
Merge pull request #269 from deeptools/file_format
Browse files Browse the repository at this point in the history 
File format
  • Loading branch information
@joachimwolff
joachimwolff committed Aug 2, 2018
2 parents 40fb89a + 95c5cfa commit b586435
Show file tree
Hide file tree
Showing 9 changed files with 74 additions and 79 deletions.
30 changes: 15 additions & 15 deletions hicexplorer/HiCMatrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,19 +427,19 @@ def convert_to_obs_exp_matrix(self, maxdepth=None, zscore=False, perchr=False):
>>> hic.matrix = csr_matrix(matrix)
>>> hic.setMatrix(hic.matrix, cut_intervals)
>>> hic.convert_to_obs_exp_matrix().todense()
matrix([[ 1. , 0.8, 1. , 1. , 0. ],
[ 0. , 4. , 1.5, 1. , 1. ],
[ 0. , 0. , 0. , 0.7, 2. ],
[ 0. , 0. , 0. , 0. , 1. ],
[ 0. , 0. , 0. , 0. , 0. ]])
matrix([[1. , 0.8, 1. , 1. , 0. ],
[0. , 4. , 1.5, 1. , 1. ],
[0. , 0. , 0. , 0.7, 2. ],
[0. , 0. , 0. , 0. , 1. ],
[0. , 0. , 0. , 0. , 0. ]])
>>> hic.matrix = csr_matrix(matrix)
>>> hic.convert_to_obs_exp_matrix(maxdepth=20).todense()
matrix([[ 1. , 0.8, 1. , 0. , 0. ],
[ 0. , 4. , 1.5, 1. , 0. ],
[ 0. , 0. , 0. , 0.7, nan],
[ 0. , 0. , 0. , 0. , nan],
[ 0. , 0. , 0. , 0. , 0. ]])
matrix([[1. , 0.8, 1. , 0. , 0. ],
[0. , 4. , 1.5, 1. , 0. ],
[0. , 0. , 0. , 0.7, nan],
[0. , 0. , 0. , 0. , nan],
[0. , 0. , 0. , 0. , 0. ]])
>>> hic.matrix = csr_matrix(matrix)
>>> hic.convert_to_obs_exp_matrix(zscore=True).todense()
Expand Down Expand Up @@ -1278,11 +1278,11 @@ def restoreMaskedBins(self):
>>> hic.restoreMaskedBins()
>>> hic.matrix.todense()
matrix([[ 0., 10., 5., 0., 0.],
[ 10., 0., 15., 0., 1.],
[ 5., 15., 0., 0., 3.],
[ 0., 0., 0., 0., 0.],
[ 0., 1., 3., 0., 0.]])
matrix([[ 0., 10., 5., 0., 0.],
[10., 0., 15., 0., 1.],
[ 5., 15., 0., 0., 3.],
[ 0., 0., 0., 0., 0.],
[ 0., 1., 3., 0., 0.]])
>>> hic.cut_intervals
[('a', 0, 10, 1), ('a', 10, 20, 1), ('a', 20, 30, 1), ('a', 30, 40, 1), ('b', 40, 50, 1)]
Expand Down
19 changes: 7 additions & 12 deletions hicexplorer/hicConvertFileFormats.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ def parse_arguments(args=None):
parserRequired.add_argument('--outputFormat',
help='Output format. The following options are available: `h5` (native HiCExplorer '
'format based on hdf5 storage format). '
' `cool` and `hic`',
' `cool` and `ginteractions`',
default='cool',
choices=['cool', 'mcool'],
choices=['cool', 'h5', 'ginteractions'],
required=True)

# parserRequired.add_argument("--modus", "-mo",
Expand Down Expand Up @@ -88,7 +88,7 @@ def main(args=None):
for matrix in args.matrices:
hic2cool_convert(matrix, args.outFileName, 0)
return
elif args.inputFormat in ['hicpro', 'homer', 'h5'] and args.outputFormat == 'cool':
elif args.inputFormat in ['hicpro', 'homer', 'h5']: # and args.outputFormat in ['cool':
if args.inputFormat == 'hicpro':
if len(args.matrices) != len(args.bedFileHicpro):
log.error('Number of matrices and associated bed files need to be the same.')
Expand All @@ -102,21 +102,16 @@ def main(args=None):
else:
matrixFileHandlerInput = MatrixFileHandler(pFileType=args.inputFormat, pMatrixFile=matrix)

matrix_, cut_intervals, nan_bins, \
_matrix, cut_intervals, nan_bins, \
correction_factors, distance_counts = matrixFileHandlerInput.load()

log.debug('self.matrix {}'.format(matrix_))
log.debug('self.nan_bins {}'.format(nan_bins))
log.debug('self.cut_intervals {}'.format(cut_intervals))
log.debug('self.correction_factors {}'.format(correction_factors))
matrixFileHandlerOutput = MatrixFileHandler(pFileType=args.outputFormat)

matrixFileHandlerOutput = MatrixFileHandler()

matrixFileHandlerOutput.set_matrix_variables(matrix_, cut_intervals, nan_bins,
matrixFileHandlerOutput.set_matrix_variables(_matrix, cut_intervals, nan_bins,
correction_factors, distance_counts)
log.debug('Setting done')

matrixFileHandlerOutput.save(matrix + '.cool', pSymmetric=True, pApplyCorrection=False)
matrixFileHandlerOutput.save(matrix + '.' + args.outputFormat, pSymmetric=True, pApplyCorrection=False)

# create hiC matrix with given input format
# additional file needed for lieberman format
Expand Down
2 changes: 1 addition & 1 deletion hicexplorer/hicMergeMatrixBins.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ def merge_bins(hic, num_bins):
run merge_matrix
>>> merge_matrix = merge_bins(hic, 2)
>>> merge_matrix.cut_intervals
[('a', 0, 20, 0.75), ('a', 20, 40, 0.55000000000000004), ('b', 40, 50, 1.0)]
[('a', 0, 20, 0.75), ('a', 20, 40, 0.55), ('b', 40, 50, 1.0)]
>>> merge_matrix.matrix.todense()
matrix([[120, 28, 1],
[ 28, 177, 4],
Expand Down
20 changes: 10 additions & 10 deletions hicexplorer/hicPlotDistVsCounts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,19 +139,19 @@ def compute_distance_mean(hicmat, maxdepth=None, perchr=False):
>>> hic.matrix = csr_matrix(matrix)
>>> hic.setMatrix(hic.matrix, cut_intervals)
>>> hic.convert_to_obs_exp_matrix().todense()
matrix([[ 1. , 0.8, 1. , 1. , 0. ],
[ 0. , 4. , 1.5, 1. , 1. ],
[ 0. , 0. , 0. , 0.7, 2. ],
[ 0. , 0. , 0. , 0. , 1. ],
[ 0. , 0. , 0. , 0. , 0. ]])
matrix([[1. , 0.8, 1. , 1. , 0. ],
[0. , 4. , 1.5, 1. , 1. ],
[0. , 0. , 0. , 0.7, 2. ],
[0. , 0. , 0. , 0. , 1. ],
[0. , 0. , 0. , 0. , 0. ]])
>>> hic.matrix = csr_matrix(matrix)
>>> hic.convert_to_obs_exp_matrix(maxdepth=20).todense()
matrix([[ 1. , 0.8, 1. , 0. , 0. ],
[ 0. , 4. , 1.5, 1. , 0. ],
[ 0. , 0. , 0. , 0.7, nan],
[ 0. , 0. , 0. , 0. , nan],
[ 0. , 0. , 0. , 0. , 0. ]])
matrix([[1. , 0.8, 1. , 0. , 0. ],
[0. , 4. , 1.5, 1. , 0. ],
[0. , 0. , 0. , 0.7, nan],
[0. , 0. , 0. , 0. , nan],
[0. , 0. , 0. , 0. , 0. ]])
>>> hic.matrix = csr_matrix(matrix)
>>> hic.convert_to_obs_exp_matrix(zscore=True).todense()
Expand Down
20 changes: 9 additions & 11 deletions hicexplorer/lib/ginteractions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,14 @@ def __init__(self, pMatrixFile):
def load(self):
log.error('Not implemented')

def save(self, fileName):
log.error('Not implemented')
def save(self, pFileName, pSymmetric=None, pApplyCorrection=None):

# self.restoreMaskedBins()
# log.debug(self.matrix.shape)
# mat_coo = triu(self.matrix, k=0, format='csr').tocoo()
# fileh = open("{}.tsv".format(fileName), 'w')
# for idx, counts in enumerate(mat_coo.data):
# chr_row, start_row, end_row, _ = self.cut_intervals[mat_coo.row[idx]]
# chr_col, start_col, end_col, _ = self.cut_intervals[mat_coo.col[idx]]
# fileh.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(chr_row, int(start_row), int(end_row),
# chr_col, int(start_col), int(end_col), counts))
# fileh.close()
log.debug(self.matrix.shape)
mat_coo = triu(self.matrix, k=0, format='csr').tocoo()
with open("{}.tsv".format(pFileName), 'w') as fileh:
for idx, counts in enumerate(mat_coo.data):
chr_row, start_row, end_row, _ = self.cut_intervals[mat_coo.row[idx]]
chr_col, start_col, end_col, _ = self.cut_intervals[mat_coo.col[idx]]
fileh.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(chr_row, int(start_row), int(end_row),
chr_col, int(start_col), int(end_col), counts))
16 changes: 8 additions & 8 deletions hicexplorer/reduceMatrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,17 +129,17 @@ def reduce_matrix(matrix, bins_to_merge, use_triu=True, diagonal=False):
>>> dia = dia_matrix(([A.diagonal()], [0]), shape=A.shape)
>>> A= csr_matrix(A + A.T - dia)
>>> print(A.todense())
[[ 0.1 0.1 0.2 0.2 nan]
[ 0.1 0.1 0.2 0.2 1.1]
[ 0.2 0.2 0.2 0.2 0. ]
[ 0.2 0.2 0.2 0.1 0. ]
[ nan 1.1 0. 0. 0. ]]
[[0.1 0.1 0.2 0.2 nan]
[0.1 0.1 0.2 0.2 1.1]
[0.2 0.2 0.2 0.2 0. ]
[0.2 0.2 0.2 0.1 0. ]
[nan 1.1 0. 0. 0. ]]
>>> ll = [(0,1), (2,3), (4,)]
>>> print(reduce_matrix(A, ll, diagonal=True, use_triu=False).todense())
[[ 0.4 0.8 nan]
[ 0.8 0.7 0. ]
[ nan 0. 0. ]]
[[0.4 0.8 nan]
[0.8 0.7 0. ]
[nan 0. 0. ]]
"""

if use_triu:
Expand Down
8 changes: 4 additions & 4 deletions hicexplorer/test/general/test_hicPlotDistVsCounts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ def test_plot():
args = "--matrices {} --plotFile {} --plotsize 6 4".format(matrix, outfile.name).split()
hicPlotDistVsCounts.main(args)

# don't using matplotlib compare images not anymore
# because matplotlib is, depending on version, implementation or context,
# slightly different images
# local computer: test passes with delta of 3000
# travis: needs to be at least 4500 to pass
# I love this voodoo :(
size_new = os.path.getsize(outfile.name)
size_reference = os.path.getsize(ROOT + 'hicPlotDistVsCounts/dist_vs_counts.png',)
assert abs(size_new - size_reference) < 3000
assert abs(size_new - size_reference) < 5000

os.remove(outfile.name)
13 changes: 7 additions & 6 deletions hicexplorer/test/general/test_hicTransform.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

ROOT = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "test_data/")
original_matrix = ROOT + "small_test_matrix_50kb_res.h5"
DELTA_DECIMAL = 2


def test_hic_transfer_all():
Expand All @@ -24,19 +25,19 @@ def test_hic_transfer_all():
# obs_exp
test = hm.hiCMatrix(ROOT + "hicTransform/obs_exp_small_50kb.h5")
new = hm.hiCMatrix(dirname_new + "/obs_exp_" + basename_new)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data, decimal=DELTA_DECIMAL)
os.unlink(dirname_new + "/obs_exp_" + basename_new)

# pearson
test = hm.hiCMatrix(ROOT + "hicTransform/pearson_small_50kb.h5")
new = hm.hiCMatrix(dirname_new + "/pearson_" + basename_new)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data, decimal=DELTA_DECIMAL)
os.unlink(dirname_new + "/pearson_" + basename_new)

# covariance
test = hm.hiCMatrix(ROOT + "hicTransform/covariance_small_50kb.h5")
new = hm.hiCMatrix(dirname_new + "/covariance_" + basename_new)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data, decimal=DELTA_DECIMAL)
os.unlink(dirname_new + "/covariance_" + basename_new)
os.unlink(outfile.name)

Expand All @@ -51,7 +52,7 @@ def test_hic_transfer_obs_exp():
test = hm.hiCMatrix(ROOT + "hicTransform/obs_exp_small_50kb.h5")

new = hm.hiCMatrix(outfile.name)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data, decimal=DELTA_DECIMAL)
os.unlink(outfile.name)


Expand All @@ -66,7 +67,7 @@ def test_hic_transfer_pearson():
test = hm.hiCMatrix(ROOT + "hicTransform/pearson_small_50kb.h5")

new = hm.hiCMatrix(outfile.name)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data, decimal=DELTA_DECIMAL)
os.unlink(outfile.name)


Expand All @@ -80,5 +81,5 @@ def test_hic_transfer_covariance():
test = hm.hiCMatrix(ROOT + "hicTransform/covariance_small_50kb.h5")

new = hm.hiCMatrix(outfile.name)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data)
nt.assert_array_almost_equal(test.matrix.data, new.matrix.data, decimal=DELTA_DECIMAL)
os.unlink(outfile.name)
25 changes: 13 additions & 12 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
numpy = 1.13.*
scipy =1.0.*
matplotlib =2.1.*
pysam >= 0.11.2
numpy = 1.14.*
scipy = 1.1.*
matplotlib = 2.2*
pysam = 0.14.*
intervaltree = 2.1.*
biopython >= 1.68
pytables = 3.3.*
pandas = 0.20.*
biopython = 1.72.*
pytables = 3.4.*
pandas = 0.23.*
pybigwig = 0.3.*
cooler = 0.7.6
jinja2 = 2.9.*
six = 1.10.*
cooler = 0.7.10
jinja2 = 2.10
six = 1.11.*
future = 0.16.*
unidecode = 0.4.*
hic2cool
unidecode = 1.0.*
hic2cool = 0.4.*
scikit-learn = 0.19.*

0 comments on commit b586435

Please sign in to comment.