Merge pull request #108 from Autoplectic/entropy_triangle

Entropy triangle

dit/abstractdist.py

@@ -340,7 +340,7 @@ def get_abstract_dist(dist):
     else:
         class D(object):
             n_variables = dist.outcome_length()
-            n_elements = np.prod(map(len, dist.alphabet))
+            n_elements = np.prod(list(map(len, dist.alphabet)))
             def parameter_array(self, indexes, cache=None):
                 return brute_marginal_array(dist, indexes, rv_mode='indexes')
         d = D()

dit/profiles/__init__.py

@@ -6,5 +6,6 @@
 """
 
 from .complexity_profile import ComplexityProfile
+from .entropy_triangle import EntropyTriangle
 from .marginal_utility_of_information import MUIProfile
 from .schneidman import SchneidmanProfile

dit/profiles/entropy_triangle.py

@@ -0,0 +1,113 @@
+"""
+The entropy triangle, from [Valverde-Albacete, Francisco Jose, and Carmen
+Pelaez-Moreno. "The Multivariate Entropy Triangle and Applications." Hybrid
+Artificial Intelligent Systems. Springer International Publishing, 2016.
+647-658].
+"""
+
+from ..distribution import BaseDistribution
+from ..distconst import product_distribution, uniform_like
+from ..multivariate import entropy, residual_entropy
+
+class EntropyTriangle(object):
+    """
+    BaseProfile
+
+    Static Attributes
+    -----------------
+    left_label : str
+        The label for the bottom axis when plotting.
+    right_label : str
+        The label for the right axis when plotting.
+    bottom_label : str
+        The label for the bottom axis when plotting.
+
+    Attributes
+    ----------
+    dists : [Distribution]
+    points : list of tuples
+
+    Methods
+    -------
+    draw
+        Plot the entropy triangle.
+    """
+
+    left_label = "$\operatorname{R}[\mathrm{dist}]$"
+    right_label = "$\operatorname{T}[\mathrm{dist}] + \operatorname{B}[\mathrm{dist}]$"
+    bottom_label = "$\Delta \operatorname{H}_{\Pi_\overline{X}}$"
+
+    def __init__(self, dists):
+        """
+        Initialize the entropy triangle.
+
+        Parameters
+        ----------
+        dists : [Distribution] or Distribution
+            The list of distributions to plot on the entropy triangle. If a single distribution is provided, it alone will be computed.
+        """
+        if isinstance(dists, BaseDistribution):
+            self.dists = [dists]
+        else:
+            self.dists = dists
+        self.points = [self._compute_point(dist) for dist in self.dists]
+
+    @staticmethod
+    def _compute_point(dist):
+        """
+        Compute the deviation from uniformity, dependence, and independence of a
+        distribution.
+
+        Parameters
+        ----------
+        dist : Distribution
+            The distribution to compute values for.
+        """
+        H_U = entropy(uniform_like(dist))
+        H_P = entropy(product_distribution(dist))
+
+        Delta = H_U - H_P
+        VI = residual_entropy(dist)
+        M = H_P - VI
+
+        return (Delta/H_U, M/H_U, VI/H_U)
+
+    def draw(self, ax=None, setup=True, marker='o', color='k'):
+        """
+        Plot the entropy triangle.
+
+        Parameters
+        ----------
+        ax : Axis or None
+            The matplotlib axis to plot on. If none is provided, one will be constructed.
+        setup : bool
+            If true, labels, tick marks, gridlines, and a boundary will be added to the plot. Defaults to True.
+        marker : str
+            The matplotlib marker shape to use.
+        color : str
+            The color of marker to use.
+        """
+        import ternary
+
+        if ax is None:
+            fig, ax = ternary.figure()
+            fig.set_size_inches(8, 8)
+        else:
+            ax = ternary.TernaryAxesSubplot(ax=ax)
+
+        if setup:
+            ax.boundary()
+            ax.gridlines(multiple=0.1)
+
+            fontsize = 20
+            ax.set_title("Entropy Triangle", fontsize=fontsize)
+            ax.left_axis_label(self.left_label, fontsize=fontsize)
+            ax.right_axis_label(self.right_label, fontsize=fontsize)
+            ax.bottom_axis_label(self.bottom_label, fontsize=fontsize)
+
+            ax.ticks(axis='lbr', multiple=0.1, linewidth=1)
+            ax.clear_matplotlib_ticks()
+
+        ax.scatter(self.points, marker=marker, color=color)
+
+        return ax

dit/profiles/marginal_utility_of_information.py

@@ -149,7 +149,7 @@ def _compute(self):
 
     def draw(self, ax=None): # pragma: no cover
         ax = super(MUIProfile, self).draw(ax=ax)
-        pnts = np.arange(int(self.profile.keys()[-1] + self.widths[-1]) + 1)
+        pnts = np.arange(int(max(self.profile.keys()) + self.widths[-1]) + 1)
         ax.set_xticks(pnts)
         ax.set_xticklabels(pnts)
         return ax

docs/doc_environment.yml

@@ -13,6 +13,7 @@ dependencies:
   - pip:
     - ipython[nbconvert] == 4.0.1
     - iterutils >= 0.1.6
+    - python-ternary
     - releases
     - sarge
     - sphinxcontrib-bibtex

docs/profiles.rst

@@ -81,6 +81,8 @@ The I-diagrams for these four examples can be computed thusly:
    | I[0:1:2] | -1.000 |
    +----------+--------+
 
+.. py:class:: dit.profiles.ComplexityProfile
+
 Complexity Profile
 ==================
 
@@ -114,6 +116,8 @@ Finally, example 4 (where each variable is the ``exclusive or`` of the other two
    @savefig complexity_profile_example_4.png width=500 align=center
    In [14]: ComplexityProfile(ex4).draw();
 
+.. py:class:: dit.profiles.MUIProfile
+
 Marginal Utility of Information
 ===============================
 
@@ -147,6 +151,8 @@ Lastly, the ``xor`` example:
    @savefig mui_profile_example_4.png width=500 align=center
    In [18]: MUIProfile(ex4).draw();
 
+.. py:class:: dit.profiles.SchneidmanProfile
+
 Schneidman Profile
 ==================
 
@@ -182,3 +188,47 @@ And for the ``xor``, all bits appear independent until fixing the three-way marg
 
    @savefig schneidman_profile_example_4.png width=500 align=center
    In [22]: SchneidmanProfile(ex4).draw();
+
+.. py:class:: dit.profiles.EntropyTriangle
+
+Entropy Triangle
+================
+
+The entropy triangle :cite:`valverde2016multivariate` is a method of visualizing how the information in the distribution is distributed among deviation from uniformity, independence, and dependence. The deviation from independence is measured by considering the difference in entropy between a independent variables with uniform distributions, and independent variables with the same marginal distributions as the distribution in question. Independence is measured via the :doc:`measures/multivariate/residual_entropy`, and dependence is measured by the sum of the :doc:`measures/multivariate/total_correlation` and :doc:`measures/multivariate/binding_information`.
+
+All four examples lay along the left axis because their distributions are uniform over the events that have non-zero probability.
+
+In the first example, the distribution is all independence because the three variables are, in fact, independent:
+
+.. ipython::
+
+   @savefig entropy_triangle_example_1.png width=500 align=center
+   In [23]: EntropyTriangle(ex1).draw();
+
+In the second example, the distribution is all dependence, because the three variables are perfectly entwined:
+
+.. ipython::
+
+   @savefig entropy_triangle_example_2.png width=500 align=center
+   In [24]: EntropyTriangle(ex2).draw();
+
+Here, there is a mix of independence and dependence:
+
+.. ipython::
+
+   @savefig entropy_triangle_example_3.png width=500 align=center
+   In [25]: EntropyTriangle(ex3).draw();
+
+And finally, in the case of ``xor``, the variables are completely dependent again:
+
+.. ipython::
+
+   @savefig entropy_triangle_example_4.png width=500 align=center
+   In [26]: EntropyTriangle(ex4).draw();
+
+We can also plot all four on the same entropy triangle:
+
+.. ipython::
+
+   @savefig entropy_triangle_all_examples.png width=500 align=center
+   In [27]: EntropyTriangle([ex1, ex2, ex3, ex4]).draw();

docs/references.bib

@@ -187,3 +187,12 @@ @article{Lamberti2004
   year={2004},
   publisher={Elsevier}
 }
+
+@incollection{valverde2016multivariate,
+  title={The Multivariate Entropy Triangle and Applications},
+  author={Valverde-Albacete, Francisco Jos{\'e} and Pel{\'a}ez-Moreno, Carmen},
+  booktitle={Hybrid Artificial Intelligent Systems},
+  pages={647--658},
+  year={2016},
+  publisher={Springer}
+}

requirements_optional.txt

@@ -1,5 +1,7 @@
 Cython
 cvxopt
 ipython
+matplotlib
 numdifftools
+python-ternary
 scipy >= 0.15.0