Merge pull request #22 from beringresearch/ivis-supvised-softmax

Ivis supvised softmax

.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 *.swp
 .ipynb_checkpoints
 _build
+*.pyc

docs/auto_examples/index.rst

@@ -13,7 +13,27 @@ Several examples of how ivis can be used in common machine learning tasks.
 
 .. raw:: html
 
-    <div class="sphx-glr-thumbcontainer" tooltip="Example of reducing dimensionality of the iris dataset using ivis. ">
+    <div class="sphx-glr-thumbcontainer" tooltip="Ivis can be applied easily applied to unstructured datasets, including images. Here we visualis...">
+
+.. only:: html
+
+    .. figure:: /auto_examples/images/thumb/sphx_glr_mnist_thumb.png
+
+        :ref:`sphx_glr_auto_examples_mnist.py`
+
+.. raw:: html
+
+    </div>
+
+
+.. toctree::
+   :hidden:
+
+   /auto_examples/mnist
+
+.. raw:: html
+
+    <div class="sphx-glr-thumbcontainer" tooltip="Example of reducing dimensionality of the iris dataset using ivis.">
 
 .. only:: html
 

docs/auto_examples/iris.ipynb

@@ -15,7 +15,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "\niris dataset\n============\n\nExample of reducing dimensionality of the iris dataset using ivis.\n\n"
+        "\niris dataset\n============\n\nExample of reducing dimensionality of the iris dataset using ivis.\n"
       ]
     },
     {
@@ -26,7 +26,7 @@
       },
       "outputs": [],
       "source": [
-        "import seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.datasets import load_iris\nfrom sklearn.preprocessing import MinMaxScaler\n\nfrom ivis import Ivis\n\nsns.set(context='paper', style='white')\n\nX = load_iris().data\nX = MinMaxScaler().fit_transform(X)\n\nivis = Ivis(k=3, batch_size=120, model='maaten')\nivis.fit(X)\n\nembeddings = ivis.transform(X)\n\nfig, ax = plt.subplots(figsize=(12, 10))\nplt.scatter(embeddings[:, 0],\n            embeddings[:, 1],\n            c=load_iris().target, cmap='Spectral', s=1)\nplt.setp(ax, xticks=[], yticks=[])\nplt.title('ivis embeddings of the iris dataset', fontsize=18)\n\nplt.show()"
+        "import seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.datasets import load_iris\nfrom sklearn.preprocessing import MinMaxScaler\n\nfrom ivis import Ivis\n\nsns.set(context='paper', style='white')\n\nX = load_iris().data\nX = MinMaxScaler().fit_transform(X)\n\nivis = Ivis(k=5, model='maaten', verbose=0)\nivis.fit(X)\n\nembeddings = ivis.transform(X)\n\nplt.figure(figsize=(5, 5), dpi=100)\nplt.scatter(embeddings[:, 0],\n            embeddings[:, 1],\n            c=load_iris().target, s=20)\nplt.xlabel('ivis 1')\nplt.ylabel('ivis 2')\nplt.title('ivis embeddings of the iris dataset')\n\nplt.show()"
       ]
     }
   ],
@@ -46,7 +46,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.7"
+      "version": "3.7.3"
     }
   },
   "nbformat": 4,

docs/auto_examples/iris.py

@@ -17,16 +17,17 @@
 X = load_iris().data
 X = MinMaxScaler().fit_transform(X)
 
-ivis = Ivis(k=3, batch_size=120, model='maaten')
+ivis = Ivis(k=5, model='maaten', verbose=0)
 ivis.fit(X)
 
 embeddings = ivis.transform(X)
 
-fig, ax = plt.subplots(figsize=(12, 10))
+plt.figure(figsize=(5, 5), dpi=100)
 plt.scatter(embeddings[:, 0],
             embeddings[:, 1],
-            c=load_iris().target, cmap='Spectral', s=1)
-plt.setp(ax, xticks=[], yticks=[])
-plt.title('ivis embeddings of the iris dataset', fontsize=18)
+            c=load_iris().target, s=20)
+plt.xlabel('ivis 1')
+plt.ylabel('ivis 2')
+plt.title('ivis embeddings of the iris dataset')
 
 plt.show()

docs/auto_examples/iris.rst

@@ -12,7 +12,6 @@ iris dataset
 
 Example of reducing dimensionality of the iris dataset using ivis.
 
-
 .. code-block:: default
 
 
@@ -28,17 +27,18 @@ Example of reducing dimensionality of the iris dataset using ivis.
     X = load_iris().data
     X = MinMaxScaler().fit_transform(X)
 
-    ivis = Ivis(k=3, batch_size=120, model='maaten')
+    ivis = Ivis(k=5, model='maaten', verbose=0)
     ivis.fit(X)
 
     embeddings = ivis.transform(X)
 
-    fig, ax = plt.subplots(figsize=(12, 10))
+    plt.figure(figsize=(5, 5), dpi=100)
     plt.scatter(embeddings[:, 0],
                 embeddings[:, 1],
-                c=load_iris().target, cmap='Spectral', s=1)
-    plt.setp(ax, xticks=[], yticks=[])
-    plt.title('ivis embeddings of the iris dataset', fontsize=18)
+                c=load_iris().target, s=20)
+    plt.xlabel('ivis 1')
+    plt.ylabel('ivis 2')
+    plt.title('ivis embeddings of the iris dataset')
 
     plt.show()
 

docs/auto_examples/mnist.ipynb

@@ -0,0 +1,54 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "%matplotlib inline"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "\nApplying ivis to the MNIST dataset\n==================================\n\nIvis can be applied easily applied to unstructured datasets, including images.\nHere we visualise the MNSIT digits dataset using two-dimensional ivis\nembeddings.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "collapsed": false
+      },
+      "outputs": [],
+      "source": [
+        "import os\nimport matplotlib.pyplot as plt\n\nfrom sklearn.datasets import fetch_openml\nfrom ivis import Ivis\n\nmnist = fetch_openml('mnist_784', version=1)\n\nivis = Ivis(model='maaten', verbose=0)\nembeddings = ivis.fit_transform(mnist.data)\n\ncolor = mnist.target.astype(int)\n\nplt.figure(figsize=(8, 8), dpi=150)\nplt.scatter(x=embeddings[:, 0],\n            y=embeddings[:, 1], c=color, cmap=\"Spectral\", s=0.1)\nplt.xlabel('ivis 1')\nplt.ylabel('ivis 2')\nplt.show()\n\nos.remove('annoy.index')"
+      ]
+    }
+  ],
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.7.3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

docs/auto_examples/mnist.py

@@ -0,0 +1,30 @@
+"""
+Applying ivis to the MNIST dataset
+==================================
+
+Ivis can be applied easily applied to unstructured datasets, including images.
+Here we visualise the MNSIT digits dataset using two-dimensional ivis
+embeddings.
+"""
+
+import os
+import matplotlib.pyplot as plt
+
+from sklearn.datasets import fetch_openml
+from ivis import Ivis
+
+mnist = fetch_openml('mnist_784', version=1)
+
+ivis = Ivis(model='maaten', verbose=0)
+embeddings = ivis.fit_transform(mnist.data)
+
+color = mnist.target.astype(int)
+
+plt.figure(figsize=(8, 8), dpi=150)
+plt.scatter(x=embeddings[:, 0],
+            y=embeddings[:, 1], c=color, cmap="Spectral", s=0.1)
+plt.xlabel('ivis 1')
+plt.ylabel('ivis 2')
+plt.show()
+
+os.remove('annoy.index')

docs/auto_examples/mnist.rst

@@ -0,0 +1,73 @@
+.. note::
+    :class: sphx-glr-download-link-note
+
+    Click :ref:`here <sphx_glr_download_auto_examples_mnist.py>` to download the full example code
+.. rst-class:: sphx-glr-example-title
+
+.. _sphx_glr_auto_examples_mnist.py:
+
+
+Applying ivis to the MNIST dataset
+==================================
+
+Ivis can be applied easily applied to unstructured datasets, including images.
+Here we visualise the MNSIT digits dataset using two-dimensional ivis
+embeddings.
+
+.. code-block:: default
+
+
+    import os
+    import matplotlib.pyplot as plt
+
+    from sklearn.datasets import fetch_openml
+    from ivis import Ivis
+
+    mnist = fetch_openml('mnist_784', version=1)
+
+    ivis = Ivis(model='maaten', verbose=0)
+    embeddings = ivis.fit_transform(mnist.data)
+
+    color = mnist.target.astype(int)
+
+    plt.figure(figsize=(8, 8), dpi=150)
+    plt.scatter(x=embeddings[:, 0],
+                y=embeddings[:, 1], c=color, cmap="Spectral", s=0.1)
+    plt.xlabel('ivis 1')
+    plt.ylabel('ivis 2')
+    plt.show()
+
+    os.remove('annoy.index')
+
+
+.. rst-class:: sphx-glr-timing
+
+   **Total running time of the script:** ( 0 minutes  0.000 seconds)
+
+
+.. _sphx_glr_download_auto_examples_mnist.py:
+
+
+.. only :: html
+
+ .. container:: sphx-glr-footer
+    :class: sphx-glr-footer-example
+
+
+
+  .. container:: sphx-glr-download
+
+     :download:`Download Python source code: mnist.py <mnist.py>`
+
+
+
+  .. container:: sphx-glr-download
+
+     :download:`Download Jupyter notebook: mnist.ipynb <mnist.ipynb>`
+
+
+.. only:: html
+
+ .. rst-class:: sphx-glr-signature
+
+    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.readthedocs.io>`_

docs/auto_examples/sklearn_pipeline.ipynb

@@ -15,7 +15,7 @@
       "cell_type": "markdown",
       "metadata": {},
       "source": [
-        "\nIntegrating ivis with standard sklearn pipelines\n================================================\n\n`Ivis` class extends sklearn's `BaseEstimator`, making it easy to incorporate ivis into a standard classification or regression pipeline.\n\n"
+        "\nIntegrating ivis with standard sklearn pipelines\n================================================\n\n`Ivis` class extends sklearn's `BaseEstimator`, making it easy to incorporate ivis into a standard classification or regression pipeline.\n"
       ]
     },
     {
@@ -46,7 +46,7 @@
       "name": "python",
       "nbconvert_exporter": "python",
       "pygments_lexer": "ipython3",
-      "version": "3.6.7"
+      "version": "3.7.3"
     }
   },
   "nbformat": 4,

docs/auto_examples/sklearn_pipeline.rst

@@ -12,7 +12,6 @@ Integrating ivis with standard sklearn pipelines
 
 `Ivis` class extends sklearn's `BaseEstimator`, making it easy to incorporate ivis into a standard classification or regression pipeline.
 
-
 .. code-block:: default
 
 

docs/index.rst

@@ -14,7 +14,7 @@ ivis: structure preserving dimensionality reduction
 .. |fig2| image:: _static/ivis_retinal_bipolar_cells.png
   :width: 49 %
 
-ivis is a machine learning algorithm for reducing dimensionality of very large datasets. ivis preserves global data structures in a low-dimensional space, adds new data points to existing embeddings using a parametric mapping function, and scales linearly to millions of observations.
+``ivis`` is a machine learning algorithm for reducing dimensionality of very large datasets. ``ivis`` preserves global data structures in a low-dimensional space, adds new data points to existing embeddings using a parametric mapping function, and scales linearly to millions of observations. The algorithm is described in detail in `Structure-preserving visualisation of high dimensional single-cell datasets <https://www.nature.com/articles/s41598-019-45301-0>`_.
 
 The latest development version is on `github <https://github.com/beringresearch/ivis>`_.
 
@@ -25,6 +25,7 @@ The latest development version is on `github <https://github.com/beringresearch/
    Installation <installation>
    Quickstart <quickstart>
    Hyperparameter Selection <hyperparameters>
+   Supervised ivis <supervised>
    Examples <auto_examples/index>
 
 .. toctree::

docs/supervised.rst

@@ -0,0 +1,62 @@
+.. _supervised:
+
+Supervised Dimensionality Reduction
+===================================
+
+``ivis`` is able to make use of any provided class labels to perform
+supervised dimensionality reduction. Supervised embeddings
+combine the distance-based characteristics of the unsupervised ``ivis``
+algorithm with clear class boundaries between the class categories. The
+resulting embeddings encode relevant class-specific information into
+lower dimensional space, making them useful for enhancing the
+performance of a classifier.
+
+To train ``ivis`` in supervised mode, simply provide the labels to the
+fit method's ``Y`` parameter. These labels should be a list of 0-indexed
+integers with each integer corresponding to a class.
+
+::
+
+    from keras.datasets import mnist
+    import numpy as np
+
+    (X_train, Y_train), (X_test, Y_test)  = mnist.load_data()
+
+    # Rescale to 0-1
+    X_train = X_train / 255.
+    X_test = X_test / 255.
+
+    # Flatten images to 1D vectors
+    X_train = np.reshape(X_train, (len(X_train), 28 * 28))
+    X_test = np.reshape(X_test, (len(X_test), 28 * 28))
+
+    model = Ivis(n_epochs_without_progress=5)
+    model.fit(X_train, Y_train)
+
+Experimental data has shown that ``ivis`` converges to a solution faster
+in supervised mode. Therefore, our suggestion is to lower the value of
+the ``n_epochs_without_progress`` parameter from the default of 50 to
+around 5. Here are the resulting embeddings:
+
+.. image:: _static/mnist-embedding-comparison_titled.png
+
+It is possible to control the relative importance Ivis places on the
+labels when training in supervised mode with the
+``classification_weight`` parameter. This variable should be a float
+between 0.0 to 1.0, with higher values resulting in classification
+affecting the training process more, and smaller values resulting in it
+impacting the training less. By default, the parameter is set to 0.5.
+Increasing it to 0.8 will result in more cleanly separated classes.
+
+::
+
+    weight = 0.8
+    model = Ivis(n_epochs_without_progress=5,
+                 classification_weight=weight)
+    model.fit(X_train, Y_train)
+
+As an illustration of the impact the ``classification_weight`` has on 
+the resulting embeddings, see the following plot of supervised ``ivis`` 
+applied to MNIST with different weight values:
+
+.. image:: _static/classification-weight-impact-mnist.png

ivis/__init__.py

@@ -1,3 +1,5 @@
 from . import data
 from . import nn
-from .ivis import Ivis
+from .ivis import Ivis
+
+__version__ = '1.2.0'