draw_plot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "from matplotlib.ticker import NullFormatter\n",
    "\n",
    "from scipy.sparse import csr_matrix, lil_matrix, vstack\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.manifold import TSNE\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk import word_tokenize\n",
    "from nlp import load_dataset\n",
    "import os\n",
    "#nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_FOLDER = 'data'\n",
    "dataset = load_dataset('de_politik_news.py', cache_dir=DATA_FOLDER)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_set = set()\n",
    "for text in dataset['validation']['text']:\n",
    "    word_set.update(word_tokenize(text.lower()))\n",
    "for text in dataset['test']['text']:\n",
    "    word_set.update(word_tokenize(text.lower()))\n",
    "\n",
    "word_dict =  { word : i for i,word in enumerate(list(word_set))}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create BOW validation vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_valid = lil_matrix((len(dataset['validation']['text']), len(word_set)), dtype=np.int8)\n",
    "for i, text in enumerate(dataset['validation']['text']):\n",
    "    for word in word_tokenize(text.lower()):\n",
    "        if word in word_dict:\n",
    "            embeddings_valid[i, word_dict[word]] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create BOW test vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings_test = lil_matrix((len(dataset['test']['text']), len(word_set)), dtype=np.int8)\n",
    "for i, text in enumerate(dataset['test']['text']):\n",
    "    for word in word_tokenize(text.lower()):\n",
    "        if word in word_dict:\n",
    "            embeddings_test[i, word_dict[word]] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reduce dimentions via SVD and t-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = vstack([embeddings_test, embeddings_valid])\n",
    "classes = dataset['test']['class']+ dataset['validation']['class']\n",
    "svd = TruncatedSVD(n_components=2)\n",
    "svd_results_bow = svd.fit_transform(embeddings)\n",
    "\n",
    "svd = TruncatedSVD(n_components=1024)\n",
    "svd_long_bow = svd.fit_transform(embeddings)\n",
    "tsne = TSNE(n_components=2, random_state=0)\n",
    "tsne_results_bow = tsne.fit_transform(svd_long_bow)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot data with classes shown different colors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_svd_and_t_sne():\n",
    "    class_color_dict = {'far-left':0, 'center-left':1, 'center':2, 'center-right':3, 'far-right':4}\n",
    "    class_color = [class_color_dict[i] for i in classes]#dataset['test']['class']]\n",
    "    \n",
    "    plt.figure(figsize=(15, 8))\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.title('SVD',fontsize=20,color='black')\n",
    "    svd_plot = plt.scatter(svd_results_bow[:, 0], svd_results_bow[:, 1],\n",
    "                           c=class_color, label = classes)#dataset['test']['class'])\n",
    "    plt.legend(handles=svd_plot.legend_elements()[0], loc='upper left',\n",
    "           labels=['far-left', 'center-left', 'center','center-right','far-right'],fontsize=15)\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.title('t-SNE (over SVD)',fontsize=20)\n",
    "    tsne_plot = plt.scatter(tsne_results_bow[:, 0], tsne_results_bow[:, 1],\n",
    "                            c=class_color, label = classes)#dataset['validation']['class'])\n",
    "    plt.legend(handles=tsne_plot.legend_elements()[0], \n",
    "           labels=['far-left', 'center-left', 'center','center-right','far-right'],fontsize=15)\n",
    "    plt.show()\n",
    "    plt.clf()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with plt.style.context('fivethirtyeight'):\n",
    "    plot_svd_and_t_sne()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_plot():\n",
    "    \n",
    "    class_color_dict = {'far-left':0, 'center-left':1, 'center':2, 'center-right':3, 'far-right':4}\n",
    "    class_color = [class_color_dict[i] for i in classes]#dataset['test']['class']]\n",
    "    \n",
    "    fig, ax = plt.subplots(figsize=(12, 12))\n",
    "\n",
    "    ax.set_title('German political news visualization', size=20)\n",
    "\n",
    "    scatter_plt = ax.scatter(tsne_results_bow[:, 0], tsne_results_bow[:, 1],\n",
    "                            c=class_color, s=70, label = classes)#dataset['test']['class'])\n",
    "    ax.legend(handles=scatter_plt.legend_elements()[0], \n",
    "           labels=['far-left', 'center-left', 'center','center-right','far-right'],fontsize=15)\n",
    "\n",
    "    ax.tick_params(axis=\"x\", labelsize=0)\n",
    "    ax.tick_params(axis=\"y\", labelsize=0)\n",
    "    \n",
    "    ax.set_ylim(ymin=-85, ymax=85)\n",
    "    ax.set_xlim(xmin=-85, xmax=85)\n",
    "\n",
    "    plt.show()\n",
    "    plt.clf()\n",
    "    \n",
    "with plt.style.context('fivethirtyeight'):\n",
    "    draw_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}