Releasing 0.2.1

README.rst

@@ -1,10 +1,19 @@
 .. -*- mode: rst -*-
 
-|Travis|_ |AppVeyor|_
+|Travis|_ |AppVeyor|_ |Coveralls|_
 
 .. |Travis| image:: https://api.travis-ci.org/hiclib/iced.png?branch=master
 .. _Travis: https://travis-ci.org/hiclib/iced
 
+.. |AppVeyor| image::
+   https://ci.appveyor.com/api/projects/status/github/hiclib/iced?branch=master&svg=true
+.. _AppVeyor:
+   https://ci.appveyor.com/project/sklearn-ci/scikit-learn/history
+
+.. |Coveralls| image::
+   https://coveralls.io/repos/hiclib/iced/badge.svg?branch=master&service=github
+.. _Coveralls: https://coveralls.io/r/hiclib/iced
+
 
 iced
 ====

continuous_integration/install.sh

@@ -58,12 +58,12 @@ create_new_conda_env() {
 
     # Use the miniconda installer for faster download / install of conda
     # itself
-    wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh \
+    wget http://repo.continuum.io/miniconda/Miniconda-3.6.0-Linux-x86_64.sh \
         -O miniconda.sh
     chmod +x miniconda.sh && ./miniconda.sh -b
     export PATH=/home/travis/miniconda/bin:$PATH
     conda update --yes conda
-
+ 
     # Configure the conda environment and put it in the path using the
     # provided versions
     REQUIREMENTS=$(print_conda_requirements)

doc/.gitignore

@@ -1 +1,2 @@
 _build/*
+auto_examples/*

doc/Makefile

@@ -177,3 +177,6 @@ pseudoxml:
 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
 	@echo
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+deploy:
+	rsync -avz _build/html/ nvaroquaux@cbio.ensmp.fr:public_html/iced/

doc/conf.py

@@ -23,6 +23,8 @@
               'sphinx.ext.pngmath', 'numpy_ext.numpydoc']
 
 autosummary_generate = True
+autodoc_default_flags = ['members', 'inherited-members']
+
 
 # -- General configuration ------------------------------------------------
 
@@ -73,7 +75,7 @@
 # default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-# add_function_parentheses = True
+add_function_parentheses = False
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).

doc/modules/.gitignore

@@ -0,0 +1 @@
+generated

doc/modules/classes.rst

@@ -4,6 +4,7 @@ Reference
 
 This is the class and reference of iced.
 
+.. _base_ref:
 
 :mod:`iced.normalization`: Normalization
 ===============================================
@@ -15,14 +16,16 @@ This is the class and reference of iced.
 
 Functions
 ---------
+.. currentmodule:: iced
 
 .. autosummary::
     :toctree: generated/
     :template: function.rst
 
     normalization.ICE_normalization
+    normalization.SCN_normalization
 
-
+.. _normalization_ref:
 
 :mod:`iced.filter`: Filter
 ===============================================
@@ -35,9 +38,13 @@ Functions
 Functions
 ---------
 
+.. currentmodule:: iced
+
 .. autosummary::
     :toctree: generated/
     :template: function.rst
 
     filter.filter_low_counts
+    filter.filter_high_counts
 
+.. _filter_ref:

doc/modules/filter.rst

@@ -0,0 +1,11 @@
+.. _filter:
+
+=============
+Filtering
+=============
+
+The :mod:`filter` submodule contains utilities for filtering the contact count
+matrix prior to normalizing.
+
+
+.. currentmodule:: iced.filter

doc/modules/normalization.rst

@@ -0,0 +1,10 @@
+.. _normalization:
+
+==============
+Normalization
+==============
+
+The :mod:`normalization` submodule contains normalization methods based on
+a matrix scaling approach.
+
+.. currentmodule:: iced.normalization

iced/normalization.py

@@ -49,12 +49,13 @@ def ICE_normalization(X, SS=None, max_iter=3000, eps=1e-4, copy=True,
 
     if sparse.issparse(X):
         if not sparse.isspmatrix_csr(X):
-            X = sparse.csr_matrix(X)
+            X = sparse.csr_matrix(X, dtype="float")
         X.sort_indices()
     else:
         X[np.isnan(X)] = 0
-        X = X.astype('float')
+    X = X.astype('float')
 
+    mean = X.mean()
     m = X.shape[0]
     is_symetric_or_tri(X)
     old_dbias = None
@@ -79,7 +80,9 @@ def ICE_normalization(X, SS=None, max_iter=3000, eps=1e-4, copy=True,
             raise NotImplementedError
 
         dbias = sum_ds.reshape((m, 1))
+        # To avoid numerical instabilities
         dbias /= dbias[dbias != 0].mean()
+
         dbias[dbias == 0] = 1
         bias *= dbias
 
@@ -88,6 +91,9 @@ def ICE_normalization(X, SS=None, max_iter=3000, eps=1e-4, copy=True,
         else:
             X /= dbias * dbias.T
 
+        bias *= np.sqrt(X.mean() / mean)
+        X *= mean / X.mean()
+
         if old_dbias is not None and np.abs(old_dbias - dbias).sum() < eps:
             if verbose > 1:
                 print("break at iteration %d" % (it,))
@@ -96,6 +102,7 @@ def ICE_normalization(X, SS=None, max_iter=3000, eps=1e-4, copy=True,
         if verbose > 1 and old_dbias is not None:
             print('ICE at iteration %d %s' %
                   (it, np.abs(old_dbias - dbias).sum()))
+
         old_dbias = dbias.copy()
     if output_bias:
         return X, bias
@@ -134,6 +141,7 @@ def SCN_normalization(X, max_iter=300, eps=1e-6, copy=True):
 
     if copy:
         X = X.copy()
+    X = X.astype(float)
 
     for it in np.arange(max_iter):
         sum_X = np.sqrt((X ** 2).sum(0))

iced/scripts/ice

@@ -25,11 +25,11 @@ parser.add_argument("--filtering_perc", "-f",
                     help="Percentage of reads to filter out")
 parser.add_argument("--filter_low_counts_perc",
                     type=float,
-                    default=0.0,
+                    default=0.02,
                     help="Percentage of reads to filter out")
 parser.add_argument("--filter_high_counts_perc",
                     type=float,
-                    default=0.02,
+                    default=0,
                     help="Percentage of reads to filter out")
 parser.add_argument("--max_iter", "-m", default=100, type=int,
                     help="Maximum number of iterations")
@@ -57,7 +57,7 @@ if "--filter_low_counts_perc" in sys.argv and "--filtering_perc" in sys.argv:
     raise Warning("This two options are incompatible")
 if "--filtering_perc" is None and "--filter_low_counts_perc" not in sys.argv:
     filter_low_counts_perc = 0.02
-elif arg.filter_low_counts_perc not None:
+elif args.filter_low_counts_perc is not None:
     filter_low_counts_perc = args.filter_low_counts_perc
 
 if args.verbose:
@@ -75,7 +75,7 @@ else:
 if args.verbose:
     print "Normalizing..."
 
-if args.filtering_perc != 0:
+if filter_low_counts_perc != 0:
     counts = iced.filter.filter_low_counts(counts,
                                            percentage=filter_low_counts_perc,
                                            copy=False, sparsity=False)

iced/tests/test_normalization.py

@@ -1,7 +1,6 @@
 import numpy as np
 from scipy import sparse
 from numpy.testing import assert_array_almost_equal
-
 import nose
 
 from iced.normalization import ICE_normalization
@@ -10,18 +9,26 @@
 
 def test_ICE_normalization():
     n = 100
-    X = np.random.random((n, n))
+    random_state = np.random.RandomState(seed=42)
+    X = random_state.randint(0, 100, size=(n, n))
     X = X + X.T
     normed_X = ICE_normalization(X, eps=1e-10, max_iter=1000000)
     normed = normed_X.sum(axis=1)
     assert_array_almost_equal(normed / normed.mean(), np.ones((len(X), )),
                               decimal=0)
 
+    normed_X, bias = ICE_normalization(X, eps=1e-10, max_iter=100000,
+                                       output_bias=True)
+    assert_array_almost_equal(normed_X, X / (bias.T * bias), 6)
+
 
 def test_sparse_ICE_normalization():
     n = 100
-    X = np.random.random((n, n))
-    thres = (np.random.random((n, n)) > 0.5).astype(bool)
+    random_state = np.random.RandomState(seed=42)
+    X = random_state.randint(0, 100, size=(n, n))
+
+    thres = (random_state.rand(n, n) > 0.5).astype(bool)
+
     X[thres] = 0
     X = X + X.T
     sparse_X = sparse.csr_matrix(X)
@@ -33,8 +40,10 @@ def test_sparse_ICE_normalization():
 
 def test_sparse_ICE_normalization_triu():
     n = 100
-    X = np.random.random((n, n))
-    thres = (np.random.random((n, n)) > 0.5).astype(bool)
+    random_state = np.random.RandomState(seed=42)
+    X = random_state.randint(0, 100, size=(n, n))
+
+    thres = (random_state.rand(n, n) > 0.5).astype(bool)
     X[thres] = 0
     X = X + X.T
     sparse_X = sparse.triu(X)
@@ -43,12 +52,18 @@ def test_sparse_ICE_normalization_triu():
     X = np.triu(X)
     normed_X = ICE_normalization(sparse_X, eps=1e-10, max_iter=10)
     assert_array_almost_equal(X, sparse_X.todense())
+    # The sparse and dense version are going to be equal up to a constant
+    # factor
+
+    normed_X *= true_normed_X.mean() / normed_X.mean()
     assert_array_almost_equal(true_normed_X, np.array(normed_X.todense()))
 
 
 def test_SCN_normalization():
     n = 100
-    X = np.random.random((n, n))
+    random_state = np.random.RandomState(seed=42)
+    X = random_state.randint(0, 100, size=(n, n))
+
     normed_X = SCN_normalization(X)
     assert_array_almost_equal(np.sqrt((normed_X ** 2).sum(axis=1)),
                               np.ones((len(X), )))

setup.py

@@ -7,6 +7,7 @@
 MAINTAINER = 'Nelle Varoquaux'
 MAINTAINER_EMAIL = 'nelle.varoquaux@gmail.com'
 VERSION = '0.2.0'
+VERSION = '0.2.1'
 
 
 def configuration(parent_package='', top_path=None):