Evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message:\n",
      "“package ‘stats’ is not available (for R version 3.5.1)”Warning message:\n",
      "“package ‘stats’ is a base package, and should not be updated”"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The downloaded binary packages are in\n",
      "\t/var/folders/df/bxvn0w3d0qd_2b63dc5hpknc0000gn/T//Rtmp1EQQQ0/downloaded_packages\n"
     ]
    }
   ],
   "source": [
    "source(\"evaluate_predictions.R\")\n",
    "source(\"evaluate_backlog_metrics.R\")\n",
    "\n",
    "if (!require(forecast)){\n",
    "    installed <- T\n",
    "    install.packages(\"forecast\")\n",
    "}\n",
    "if (!require(dplyr)){\n",
    "    install.packages(\"dplyr\")\n",
    "    require(dplyr)\n",
    "}\n",
    "install.packages(\"stats\")\n",
    "library(gtools)\n",
    "library(stringr)\n",
    "install.packages(\"tidyverse\")\n",
    "require(stats)    \n",
    "\n",
    "if (!require(effsize)){\n",
    "    install.packages(\"effsize\")\n",
    "    require(effsize)\n",
    "}\n",
    "\n",
    "if (!require(esvis)){\n",
    "    install.packages(\"esvis\")\n",
    "    require(esvis)\n",
    "}\n",
    "if (!require(IRdisplay)){\n",
    "    install.packages(\"IRdisplay\")\n",
    "    require(IRdisplay)\n",
    "}\n",
    "if (!require(png)){\n",
    "    install.packages(\"png\")\n",
    "    require(png)\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”Warning message in wilcox.test.default(n_error, errors, paired = TRUE):\n",
      "“cannot compute exact p-value with zeroes”"
     ]
    }
   ],
   "source": [
    "eclipse_data_tools = c(\"eclipse\", \"data_tools\")\n",
    "eclipse_platform = c(\"eclipse\", \"platform\")\n",
    "eclipse_birt = c(\"eclipse\", \"birt\")\n",
    "eclipse_jdt = c(\"eclipse\", \"jdt\")\n",
    "eclipse_pde = c(\"eclipse\", \"pde\")\n",
    "\n",
    "mozilla_calendar = c(\"mozilla\", \"calendar\")\n",
    "mozilla_firefox = c(\"mozilla\", \"firefox\")\n",
    "mozilla_core = c(\"mozilla\", \"core\")\n",
    "mozilla_thunderbird = c(\"mozilla\", \"thunderbird\")\n",
    "\n",
    "kernel_io_storage = c(\"kernel\", \"IO_Storage\")\n",
    "kernel_networking = c(\"kernel\", \"networking\")\n",
    "kernel_file_system = c(\"kernel\", \"file_system\")\n",
    "\n",
    "open_office_writer = c(\"open_office\", \"writer\")\n",
    "open_office_calc = c(\"open_office\", \"calc\")\n",
    "open_office_draw = c(\"open_office\", \"draw\")\n",
    "\n",
    "apache_ant = c(\"apache\", \"ant\")\n",
    "apache_apache2 = c(\"apache\", \"apache2\")\n",
    "\n",
    "libre_office_writer = c(\"libre_office\", \"writer\")\n",
    "libre_office_calc = c(\"libre_office\", \"calc\")\n",
    "libre_office_draw = c(\"libre_office\", \"draw\")\n",
    "\n",
    "products = rbind(kernel_networking,\n",
    "                 eclipse_platform,\n",
    "                 eclipse_data_tools,\n",
    "                 eclipse_birt,\n",
    "                 eclipse_jdt,\n",
    "                 eclipse_pde,\n",
    "                 mozilla_calendar,\n",
    "                 mozilla_firefox,\n",
    "                 mozilla_core,\n",
    "                 mozilla_thunderbird,\n",
    "                 kernel_io_storage,\n",
    "                 kernel_file_system,\n",
    "                 open_office_writer,\n",
    "                 open_office_calc,\n",
    "                 open_office_draw,\n",
    "                 apache_ant,\n",
    "                 apache_apache2,\n",
    "                 libre_office_writer,\n",
    "                 libre_office_calc,\n",
    "                 libre_office_draw)  \n",
    "\n",
    "evaluate_predictions(products, \"ericsson\")\n",
    "\n",
    "#evaluate_backlog_metrics(products)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Improvements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>arima_th.16</th><th scope=col>arima_th.2</th><th scope=col>arima_th.32</th><th scope=col>arima_th.4</th><th scope=col>arima_th.64</th><th scope=col>arima_th.8</th><th scope=col>arima_th.all</th><th scope=col>ericsson</th><th scope=col>ets_th.16</th><th scope=col>ets_th.2</th><th scope=col>ets_th.32</th><th scope=col>ets_th.4</th><th scope=col>ets_th.64</th><th scope=col>ets_th.8</th><th scope=col>ets_th.all</th><th scope=col>naive</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>kernel_networking</th><td> -16.94</td><td> -78.69</td><td> -33.33</td><td> -65.57</td><td>-18.58 </td><td> -48.63</td><td>  -6.56</td><td>-38.25 </td><td> -18.03</td><td> -39.34</td><td> -20.77</td><td> -45.36</td><td>-16.94 </td><td> -15.85</td><td> -5.46 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>eclipse_platform</th><td>  -1.24</td><td> -56.14</td><td>   0.26</td><td> -42.33</td><td>  7.39 </td><td>  -5.19</td><td>   7.49</td><td>  5.01 </td><td>   1.32</td><td> -10.98</td><td>   3.89</td><td>  -8.25</td><td>  7.55 </td><td>  -8.25</td><td> 10.30 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>eclipse_data_tools</th><td>  -1.83</td><td> -23.89</td><td>  -1.60</td><td> -33.37</td><td>  2.97 </td><td> -13.60</td><td>   7.20</td><td>  2.63 </td><td>  -0.34</td><td> -10.06</td><td>   0.80</td><td>  -9.60</td><td>  3.20 </td><td>  -6.51</td><td>  4.34 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>eclipse_birt</th><td>   4.84</td><td> -59.17</td><td>   5.76</td><td> -33.83</td><td>  6.29 </td><td> -18.67</td><td>   2.65</td><td>  2.24 </td><td>  -2.36</td><td> -11.35</td><td>  -5.92</td><td> -12.26</td><td>  7.49 </td><td>  -3.44</td><td> 14.41 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>eclipse_jdt</th><td>  -7.05</td><td> -59.73</td><td>   9.05</td><td> -67.80</td><td> 14.66 </td><td> -34.66</td><td>  -4.38</td><td>-10.49 </td><td>  -4.50</td><td> -15.85</td><td>  11.43</td><td> -28.38</td><td> 16.40 </td><td> -22.68</td><td>  5.82 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>eclipse_pde</th><td> -10.30</td><td> -37.84</td><td> -21.80</td><td> -37.37</td><td>-12.34 </td><td> -16.53</td><td>   3.35</td><td>-15.81 </td><td>  -0.84</td><td>  -9.10</td><td>  -8.74</td><td> -13.29</td><td>-13.41 </td><td> -11.98</td><td>  1.32 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>mozilla_calendar</th><td> -28.55</td><td> -82.49</td><td> -34.88</td><td> -58.32</td><td>-23.10 </td><td> -34.28</td><td> -27.74</td><td> -0.07 </td><td> -12.12</td><td> -28.75</td><td>   1.55</td><td> -42.36</td><td>  0.40 </td><td> -22.09</td><td>  7.41 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>mozilla_firefox</th><td> -13.25</td><td>-101.30</td><td>  10.65</td><td> -60.77</td><td>  9.94 </td><td> -17.01</td><td>  14.19</td><td> -1.30 </td><td> -12.66</td><td> -22.82</td><td>   4.32</td><td> -21.39</td><td> 13.45 </td><td> -27.03</td><td> 16.41 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>mozilla_core</th><td>  -5.34</td><td> -82.63</td><td>  -1.24</td><td> -45.59</td><td> 11.66 </td><td> -20.65</td><td>  18.77</td><td>  8.65 </td><td>  -3.11</td><td> -11.50</td><td>   3.21</td><td> -12.30</td><td>  3.56 </td><td> -18.92</td><td> 13.46 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>mozilla_thunderbird</th><td>  -4.72</td><td> -93.28</td><td> -14.11</td><td> -45.37</td><td>  1.63 </td><td> -10.98</td><td>  15.15</td><td>  4.99 </td><td>  -9.30</td><td> -16.02</td><td>   5.22</td><td> -21.10</td><td>  5.94 </td><td> -13.61</td><td> 11.43 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>kernel_IO_Storage</th><td> -17.28</td><td> -43.75</td><td> -19.85</td><td> -42.65</td><td>-35.66 </td><td> -19.49</td><td>  -2.94</td><td>-17.28 </td><td>  -7.72</td><td> -11.03</td><td>  -5.88</td><td> -13.24</td><td> -4.41 </td><td>  -2.94</td><td> -1.84 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>kernel_file_system</th><td> -73.50</td><td>-111.66</td><td> -72.79</td><td>-135.34</td><td>-10.25 </td><td> -74.56</td><td>   5.30</td><td>-23.67 </td><td> -65.02</td><td> -94.70</td><td> -45.94</td><td> -77.03</td><td>-25.09 </td><td> -65.37</td><td>  3.18 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>open_office_writer</th><td> -84.85</td><td>-119.54</td><td>  20.34</td><td> -92.43</td><td>  5.25 </td><td> -76.13</td><td>   0.02</td><td>-17.57 </td><td> -70.58</td><td> -85.65</td><td>  10.94</td><td> -85.67</td><td>  5.81 </td><td> -72.59</td><td>  3.37 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>open_office_calc</th><td> -95.88</td><td>-161.13</td><td> -43.21</td><td>-119.44</td><td> 30.92 </td><td> -88.37</td><td> -24.35</td><td>-38.58 </td><td>  31.14</td><td> -99.35</td><td>  32.88</td><td> -88.22</td><td> 30.49 </td><td> -88.58</td><td> 15.46 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>open_office_draw</th><td>-218.82</td><td>-248.34</td><td>-164.94</td><td>-263.10</td><td>-45.02 </td><td>-237.64</td><td>-130.26</td><td>-83.03 </td><td>-134.69</td><td>-180.07</td><td>-216.61</td><td>-129.52</td><td>-66.42 </td><td>-153.14</td><td>-41.70 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>apache_ant</th><td>   9.17</td><td> -27.24</td><td>   9.61</td><td>   1.63</td><td>  8.95 </td><td>  11.11</td><td>  11.33</td><td> 12.78 </td><td>  12.16</td><td>  12.03</td><td>  11.55</td><td>  11.28</td><td>  8.86 </td><td>  11.28</td><td>  9.96 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>apache_apache2</th><td>  15.18</td><td> -61.74</td><td>  10.64</td><td> -58.63</td><td> 15.43 </td><td>   6.23</td><td>  20.49</td><td>-36.06 </td><td>  18.03</td><td>   4.93</td><td>  18.94</td><td>   8.17</td><td> 16.99 </td><td>  11.02</td><td>  4.54 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>libre_office_writer</th><td>   7.47</td><td> -88.95</td><td>  14.97</td><td> -30.29</td><td> 23.90 </td><td>   5.77</td><td>  28.69</td><td> 15.98 </td><td>  20.84</td><td>   9.83</td><td>  18.65</td><td>  17.30</td><td> 25.84 </td><td>   3.72</td><td> 27.41 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>libre_office_calc</th><td>  18.33</td><td> -81.95</td><td>  19.52</td><td> -15.38</td><td> 21.28 </td><td>  26.96</td><td>  35.41</td><td> 27.98 </td><td>  13.17</td><td>   9.14</td><td>  15.44</td><td>  17.03</td><td> 23.27 </td><td>   6.81</td><td> 31.84 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>libre_office_draw</th><td>   8.70</td><td> -64.19</td><td>   2.56</td><td> -34.78</td><td> 13.81 </td><td>  -3.84</td><td>  11.51</td><td> 12.28 </td><td>   4.35</td><td>  -6.39</td><td>   4.86</td><td>  -7.67</td><td>  7.67 </td><td>  -8.18</td><td> 10.23 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>mean_improvement</th><td> -25.79</td><td> -84.18</td><td> -15.22</td><td> -64.04</td><td>  1.46 </td><td> -33.51</td><td>  -0.73</td><td> -9.48 </td><td> -12.01</td><td> -30.85</td><td>  -8.01</td><td> -28.09</td><td>  2.53 </td><td> -25.42</td><td>  7.09 </td><td>0      </td></tr>\n",
       "\t<tr><th scope=row>improvements_standard_deviation</th><td>  55.71</td><td>  50.72</td><td>  42.67</td><td>  56.91</td><td> 19.82 </td><td>  56.62</td><td>  34.09</td><td> 25.51 </td><td>  37.94</td><td>  48.15</td><td>  51.72</td><td>  39.24</td><td> 21.30 </td><td>  40.46</td><td> 14.55 </td><td>0      </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllllllllllll}\n",
       "  & arima\\_th.16 & arima\\_th.2 & arima\\_th.32 & arima\\_th.4 & arima\\_th.64 & arima\\_th.8 & arima\\_th.all & ericsson & ets\\_th.16 & ets\\_th.2 & ets\\_th.32 & ets\\_th.4 & ets\\_th.64 & ets\\_th.8 & ets\\_th.all & naive\\\\\n",
       "\\hline\n",
       "\tkernel\\_networking &  -16.94 &  -78.69 &  -33.33 &  -65.57 & -18.58  &  -48.63 &   -6.56 & -38.25  &  -18.03 &  -39.34 &  -20.77 &  -45.36 & -16.94  &  -15.85 &  -5.46  & 0      \\\\\n",
       "\teclipse\\_platform &   -1.24 &  -56.14 &    0.26 &  -42.33 &   7.39  &   -5.19 &    7.49 &   5.01  &    1.32 &  -10.98 &    3.89 &   -8.25 &   7.55  &   -8.25 &  10.30  & 0      \\\\\n",
       "\teclipse\\_data\\_tools &   -1.83 &  -23.89 &   -1.60 &  -33.37 &   2.97  &  -13.60 &    7.20 &   2.63  &   -0.34 &  -10.06 &    0.80 &   -9.60 &   3.20  &   -6.51 &   4.34  & 0      \\\\\n",
       "\teclipse\\_birt &    4.84 &  -59.17 &    5.76 &  -33.83 &   6.29  &  -18.67 &    2.65 &   2.24  &   -2.36 &  -11.35 &   -5.92 &  -12.26 &   7.49  &   -3.44 &  14.41  & 0      \\\\\n",
       "\teclipse\\_jdt &   -7.05 &  -59.73 &    9.05 &  -67.80 &  14.66  &  -34.66 &   -4.38 & -10.49  &   -4.50 &  -15.85 &   11.43 &  -28.38 &  16.40  &  -22.68 &   5.82  & 0      \\\\\n",
       "\teclipse\\_pde &  -10.30 &  -37.84 &  -21.80 &  -37.37 & -12.34  &  -16.53 &    3.35 & -15.81  &   -0.84 &   -9.10 &   -8.74 &  -13.29 & -13.41  &  -11.98 &   1.32  & 0      \\\\\n",
       "\tmozilla\\_calendar &  -28.55 &  -82.49 &  -34.88 &  -58.32 & -23.10  &  -34.28 &  -27.74 &  -0.07  &  -12.12 &  -28.75 &    1.55 &  -42.36 &   0.40  &  -22.09 &   7.41  & 0      \\\\\n",
       "\tmozilla\\_firefox &  -13.25 & -101.30 &   10.65 &  -60.77 &   9.94  &  -17.01 &   14.19 &  -1.30  &  -12.66 &  -22.82 &    4.32 &  -21.39 &  13.45  &  -27.03 &  16.41  & 0      \\\\\n",
       "\tmozilla\\_core &   -5.34 &  -82.63 &   -1.24 &  -45.59 &  11.66  &  -20.65 &   18.77 &   8.65  &   -3.11 &  -11.50 &    3.21 &  -12.30 &   3.56  &  -18.92 &  13.46  & 0      \\\\\n",
       "\tmozilla\\_thunderbird &   -4.72 &  -93.28 &  -14.11 &  -45.37 &   1.63  &  -10.98 &   15.15 &   4.99  &   -9.30 &  -16.02 &    5.22 &  -21.10 &   5.94  &  -13.61 &  11.43  & 0      \\\\\n",
       "\tkernel\\_IO\\_Storage &  -17.28 &  -43.75 &  -19.85 &  -42.65 & -35.66  &  -19.49 &   -2.94 & -17.28  &   -7.72 &  -11.03 &   -5.88 &  -13.24 &  -4.41  &   -2.94 &  -1.84  & 0      \\\\\n",
       "\tkernel\\_file\\_system &  -73.50 & -111.66 &  -72.79 & -135.34 & -10.25  &  -74.56 &    5.30 & -23.67  &  -65.02 &  -94.70 &  -45.94 &  -77.03 & -25.09  &  -65.37 &   3.18  & 0      \\\\\n",
       "\topen\\_office\\_writer &  -84.85 & -119.54 &   20.34 &  -92.43 &   5.25  &  -76.13 &    0.02 & -17.57  &  -70.58 &  -85.65 &   10.94 &  -85.67 &   5.81  &  -72.59 &   3.37  & 0      \\\\\n",
       "\topen\\_office\\_calc &  -95.88 & -161.13 &  -43.21 & -119.44 &  30.92  &  -88.37 &  -24.35 & -38.58  &   31.14 &  -99.35 &   32.88 &  -88.22 &  30.49  &  -88.58 &  15.46  & 0      \\\\\n",
       "\topen\\_office\\_draw & -218.82 & -248.34 & -164.94 & -263.10 & -45.02  & -237.64 & -130.26 & -83.03  & -134.69 & -180.07 & -216.61 & -129.52 & -66.42  & -153.14 & -41.70  & 0      \\\\\n",
       "\tapache\\_ant &    9.17 &  -27.24 &    9.61 &    1.63 &   8.95  &   11.11 &   11.33 &  12.78  &   12.16 &   12.03 &   11.55 &   11.28 &   8.86  &   11.28 &   9.96  & 0      \\\\\n",
       "\tapache\\_apache2 &   15.18 &  -61.74 &   10.64 &  -58.63 &  15.43  &    6.23 &   20.49 & -36.06  &   18.03 &    4.93 &   18.94 &    8.17 &  16.99  &   11.02 &   4.54  & 0      \\\\\n",
       "\tlibre\\_office\\_writer &    7.47 &  -88.95 &   14.97 &  -30.29 &  23.90  &    5.77 &   28.69 &  15.98  &   20.84 &    9.83 &   18.65 &   17.30 &  25.84  &    3.72 &  27.41  & 0      \\\\\n",
       "\tlibre\\_office\\_calc &   18.33 &  -81.95 &   19.52 &  -15.38 &  21.28  &   26.96 &   35.41 &  27.98  &   13.17 &    9.14 &   15.44 &   17.03 &  23.27  &    6.81 &  31.84  & 0      \\\\\n",
       "\tlibre\\_office\\_draw &    8.70 &  -64.19 &    2.56 &  -34.78 &  13.81  &   -3.84 &   11.51 &  12.28  &    4.35 &   -6.39 &    4.86 &   -7.67 &   7.67  &   -8.18 &  10.23  & 0      \\\\\n",
       "\tmean\\_improvement &  -25.79 &  -84.18 &  -15.22 &  -64.04 &   1.46  &  -33.51 &   -0.73 &  -9.48  &  -12.01 &  -30.85 &   -8.01 &  -28.09 &   2.53  &  -25.42 &   7.09  & 0      \\\\\n",
       "\timprovements\\_standard\\_deviation &   55.71 &   50.72 &   42.67 &   56.91 &  19.82  &   56.62 &   34.09 &  25.51  &   37.94 &   48.15 &   51.72 &   39.24 &  21.30  &   40.46 &  14.55  & 0      \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| <!--/--> | arima_th.16 | arima_th.2 | arima_th.32 | arima_th.4 | arima_th.64 | arima_th.8 | arima_th.all | ericsson | ets_th.16 | ets_th.2 | ets_th.32 | ets_th.4 | ets_th.64 | ets_th.8 | ets_th.all | naive | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking |  -16.94 |  -78.69 |  -33.33 |  -65.57 | -18.58  |  -48.63 |   -6.56 | -38.25  |  -18.03 |  -39.34 |  -20.77 |  -45.36 | -16.94  |  -15.85 |  -5.46  | 0       | \n",
       "| eclipse_platform |   -1.24 |  -56.14 |    0.26 |  -42.33 |   7.39  |   -5.19 |    7.49 |   5.01  |    1.32 |  -10.98 |    3.89 |   -8.25 |   7.55  |   -8.25 |  10.30  | 0       | \n",
       "| eclipse_data_tools |   -1.83 |  -23.89 |   -1.60 |  -33.37 |   2.97  |  -13.60 |    7.20 |   2.63  |   -0.34 |  -10.06 |    0.80 |   -9.60 |   3.20  |   -6.51 |   4.34  | 0       | \n",
       "| eclipse_birt |    4.84 |  -59.17 |    5.76 |  -33.83 |   6.29  |  -18.67 |    2.65 |   2.24  |   -2.36 |  -11.35 |   -5.92 |  -12.26 |   7.49  |   -3.44 |  14.41  | 0       | \n",
       "| eclipse_jdt |   -7.05 |  -59.73 |    9.05 |  -67.80 |  14.66  |  -34.66 |   -4.38 | -10.49  |   -4.50 |  -15.85 |   11.43 |  -28.38 |  16.40  |  -22.68 |   5.82  | 0       | \n",
       "| eclipse_pde |  -10.30 |  -37.84 |  -21.80 |  -37.37 | -12.34  |  -16.53 |    3.35 | -15.81  |   -0.84 |   -9.10 |   -8.74 |  -13.29 | -13.41  |  -11.98 |   1.32  | 0       | \n",
       "| mozilla_calendar |  -28.55 |  -82.49 |  -34.88 |  -58.32 | -23.10  |  -34.28 |  -27.74 |  -0.07  |  -12.12 |  -28.75 |    1.55 |  -42.36 |   0.40  |  -22.09 |   7.41  | 0       | \n",
       "| mozilla_firefox |  -13.25 | -101.30 |   10.65 |  -60.77 |   9.94  |  -17.01 |   14.19 |  -1.30  |  -12.66 |  -22.82 |    4.32 |  -21.39 |  13.45  |  -27.03 |  16.41  | 0       | \n",
       "| mozilla_core |   -5.34 |  -82.63 |   -1.24 |  -45.59 |  11.66  |  -20.65 |   18.77 |   8.65  |   -3.11 |  -11.50 |    3.21 |  -12.30 |   3.56  |  -18.92 |  13.46  | 0       | \n",
       "| mozilla_thunderbird |   -4.72 |  -93.28 |  -14.11 |  -45.37 |   1.63  |  -10.98 |   15.15 |   4.99  |   -9.30 |  -16.02 |    5.22 |  -21.10 |   5.94  |  -13.61 |  11.43  | 0       | \n",
       "| kernel_IO_Storage |  -17.28 |  -43.75 |  -19.85 |  -42.65 | -35.66  |  -19.49 |   -2.94 | -17.28  |   -7.72 |  -11.03 |   -5.88 |  -13.24 |  -4.41  |   -2.94 |  -1.84  | 0       | \n",
       "| kernel_file_system |  -73.50 | -111.66 |  -72.79 | -135.34 | -10.25  |  -74.56 |    5.30 | -23.67  |  -65.02 |  -94.70 |  -45.94 |  -77.03 | -25.09  |  -65.37 |   3.18  | 0       | \n",
       "| open_office_writer |  -84.85 | -119.54 |   20.34 |  -92.43 |   5.25  |  -76.13 |    0.02 | -17.57  |  -70.58 |  -85.65 |   10.94 |  -85.67 |   5.81  |  -72.59 |   3.37  | 0       | \n",
       "| open_office_calc |  -95.88 | -161.13 |  -43.21 | -119.44 |  30.92  |  -88.37 |  -24.35 | -38.58  |   31.14 |  -99.35 |   32.88 |  -88.22 |  30.49  |  -88.58 |  15.46  | 0       | \n",
       "| open_office_draw | -218.82 | -248.34 | -164.94 | -263.10 | -45.02  | -237.64 | -130.26 | -83.03  | -134.69 | -180.07 | -216.61 | -129.52 | -66.42  | -153.14 | -41.70  | 0       | \n",
       "| apache_ant |    9.17 |  -27.24 |    9.61 |    1.63 |   8.95  |   11.11 |   11.33 |  12.78  |   12.16 |   12.03 |   11.55 |   11.28 |   8.86  |   11.28 |   9.96  | 0       | \n",
       "| apache_apache2 |   15.18 |  -61.74 |   10.64 |  -58.63 |  15.43  |    6.23 |   20.49 | -36.06  |   18.03 |    4.93 |   18.94 |    8.17 |  16.99  |   11.02 |   4.54  | 0       | \n",
       "| libre_office_writer |    7.47 |  -88.95 |   14.97 |  -30.29 |  23.90  |    5.77 |   28.69 |  15.98  |   20.84 |    9.83 |   18.65 |   17.30 |  25.84  |    3.72 |  27.41  | 0       | \n",
       "| libre_office_calc |   18.33 |  -81.95 |   19.52 |  -15.38 |  21.28  |   26.96 |   35.41 |  27.98  |   13.17 |    9.14 |   15.44 |   17.03 |  23.27  |    6.81 |  31.84  | 0       | \n",
       "| libre_office_draw |    8.70 |  -64.19 |    2.56 |  -34.78 |  13.81  |   -3.84 |   11.51 |  12.28  |    4.35 |   -6.39 |    4.86 |   -7.67 |   7.67  |   -8.18 |  10.23  | 0       | \n",
       "| mean_improvement |  -25.79 |  -84.18 |  -15.22 |  -64.04 |   1.46  |  -33.51 |   -0.73 |  -9.48  |  -12.01 |  -30.85 |   -8.01 |  -28.09 |   2.53  |  -25.42 |   7.09  | 0       | \n",
       "| improvements_standard_deviation |   55.71 |   50.72 |   42.67 |   56.91 |  19.82  |   56.62 |   34.09 |  25.51  |   37.94 |   48.15 |   51.72 |   39.24 |  21.30  |   40.46 |  14.55  | 0       | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "                                arima_th.16 arima_th.2 arima_th.32 arima_th.4\n",
       "kernel_networking                -16.94      -78.69     -33.33      -65.57   \n",
       "eclipse_platform                  -1.24      -56.14       0.26      -42.33   \n",
       "eclipse_data_tools                -1.83      -23.89      -1.60      -33.37   \n",
       "eclipse_birt                       4.84      -59.17       5.76      -33.83   \n",
       "eclipse_jdt                       -7.05      -59.73       9.05      -67.80   \n",
       "eclipse_pde                      -10.30      -37.84     -21.80      -37.37   \n",
       "mozilla_calendar                 -28.55      -82.49     -34.88      -58.32   \n",
       "mozilla_firefox                  -13.25     -101.30      10.65      -60.77   \n",
       "mozilla_core                      -5.34      -82.63      -1.24      -45.59   \n",
       "mozilla_thunderbird               -4.72      -93.28     -14.11      -45.37   \n",
       "kernel_IO_Storage                -17.28      -43.75     -19.85      -42.65   \n",
       "kernel_file_system               -73.50     -111.66     -72.79     -135.34   \n",
       "open_office_writer               -84.85     -119.54      20.34      -92.43   \n",
       "open_office_calc                 -95.88     -161.13     -43.21     -119.44   \n",
       "open_office_draw                -218.82     -248.34    -164.94     -263.10   \n",
       "apache_ant                         9.17      -27.24       9.61        1.63   \n",
       "apache_apache2                    15.18      -61.74      10.64      -58.63   \n",
       "libre_office_writer                7.47      -88.95      14.97      -30.29   \n",
       "libre_office_calc                 18.33      -81.95      19.52      -15.38   \n",
       "libre_office_draw                  8.70      -64.19       2.56      -34.78   \n",
       "mean_improvement                 -25.79      -84.18     -15.22      -64.04   \n",
       "improvements_standard_deviation   55.71       50.72      42.67       56.91   \n",
       "                                arima_th.64 arima_th.8 arima_th.all ericsson\n",
       "kernel_networking               -18.58       -48.63      -6.56      -38.25  \n",
       "eclipse_platform                  7.39        -5.19       7.49        5.01  \n",
       "eclipse_data_tools                2.97       -13.60       7.20        2.63  \n",
       "eclipse_birt                      6.29       -18.67       2.65        2.24  \n",
       "eclipse_jdt                      14.66       -34.66      -4.38      -10.49  \n",
       "eclipse_pde                     -12.34       -16.53       3.35      -15.81  \n",
       "mozilla_calendar                -23.10       -34.28     -27.74       -0.07  \n",
       "mozilla_firefox                   9.94       -17.01      14.19       -1.30  \n",
       "mozilla_core                     11.66       -20.65      18.77        8.65  \n",
       "mozilla_thunderbird               1.63       -10.98      15.15        4.99  \n",
       "kernel_IO_Storage               -35.66       -19.49      -2.94      -17.28  \n",
       "kernel_file_system              -10.25       -74.56       5.30      -23.67  \n",
       "open_office_writer                5.25       -76.13       0.02      -17.57  \n",
       "open_office_calc                 30.92       -88.37     -24.35      -38.58  \n",
       "open_office_draw                -45.02      -237.64    -130.26      -83.03  \n",
       "apache_ant                        8.95        11.11      11.33       12.78  \n",
       "apache_apache2                   15.43         6.23      20.49      -36.06  \n",
       "libre_office_writer              23.90         5.77      28.69       15.98  \n",
       "libre_office_calc                21.28        26.96      35.41       27.98  \n",
       "libre_office_draw                13.81        -3.84      11.51       12.28  \n",
       "mean_improvement                  1.46       -33.51      -0.73       -9.48  \n",
       "improvements_standard_deviation  19.82        56.62      34.09       25.51  \n",
       "                                ets_th.16 ets_th.2 ets_th.32 ets_th.4 ets_th.64\n",
       "kernel_networking                -18.03    -39.34   -20.77    -45.36  -16.94   \n",
       "eclipse_platform                   1.32    -10.98     3.89     -8.25    7.55   \n",
       "eclipse_data_tools                -0.34    -10.06     0.80     -9.60    3.20   \n",
       "eclipse_birt                      -2.36    -11.35    -5.92    -12.26    7.49   \n",
       "eclipse_jdt                       -4.50    -15.85    11.43    -28.38   16.40   \n",
       "eclipse_pde                       -0.84     -9.10    -8.74    -13.29  -13.41   \n",
       "mozilla_calendar                 -12.12    -28.75     1.55    -42.36    0.40   \n",
       "mozilla_firefox                  -12.66    -22.82     4.32    -21.39   13.45   \n",
       "mozilla_core                      -3.11    -11.50     3.21    -12.30    3.56   \n",
       "mozilla_thunderbird               -9.30    -16.02     5.22    -21.10    5.94   \n",
       "kernel_IO_Storage                 -7.72    -11.03    -5.88    -13.24   -4.41   \n",
       "kernel_file_system               -65.02    -94.70   -45.94    -77.03  -25.09   \n",
       "open_office_writer               -70.58    -85.65    10.94    -85.67    5.81   \n",
       "open_office_calc                  31.14    -99.35    32.88    -88.22   30.49   \n",
       "open_office_draw                -134.69   -180.07  -216.61   -129.52  -66.42   \n",
       "apache_ant                        12.16     12.03    11.55     11.28    8.86   \n",
       "apache_apache2                    18.03      4.93    18.94      8.17   16.99   \n",
       "libre_office_writer               20.84      9.83    18.65     17.30   25.84   \n",
       "libre_office_calc                 13.17      9.14    15.44     17.03   23.27   \n",
       "libre_office_draw                  4.35     -6.39     4.86     -7.67    7.67   \n",
       "mean_improvement                 -12.01    -30.85    -8.01    -28.09    2.53   \n",
       "improvements_standard_deviation   37.94     48.15    51.72     39.24   21.30   \n",
       "                                ets_th.8 ets_th.all naive\n",
       "kernel_networking                -15.85   -5.46     0    \n",
       "eclipse_platform                  -8.25   10.30     0    \n",
       "eclipse_data_tools                -6.51    4.34     0    \n",
       "eclipse_birt                      -3.44   14.41     0    \n",
       "eclipse_jdt                      -22.68    5.82     0    \n",
       "eclipse_pde                      -11.98    1.32     0    \n",
       "mozilla_calendar                 -22.09    7.41     0    \n",
       "mozilla_firefox                  -27.03   16.41     0    \n",
       "mozilla_core                     -18.92   13.46     0    \n",
       "mozilla_thunderbird              -13.61   11.43     0    \n",
       "kernel_IO_Storage                 -2.94   -1.84     0    \n",
       "kernel_file_system               -65.37    3.18     0    \n",
       "open_office_writer               -72.59    3.37     0    \n",
       "open_office_calc                 -88.58   15.46     0    \n",
       "open_office_draw                -153.14  -41.70     0    \n",
       "apache_ant                        11.28    9.96     0    \n",
       "apache_apache2                    11.02    4.54     0    \n",
       "libre_office_writer                3.72   27.41     0    \n",
       "libre_office_calc                  6.81   31.84     0    \n",
       "libre_office_draw                 -8.18   10.23     0    \n",
       "mean_improvement                 -25.42    7.09     0    \n",
       "improvements_standard_deviation   40.46   14.55     0    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "improvements_evaluation = read.csv(file=paste(\"data/improvements_evaluation_week_1.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "improvements_evaluation\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th></th><th scope=col>arima_th.16</th><th scope=col>arima_th.2</th><th scope=col>arima_th.32</th><th scope=col>arima_th.4</th><th scope=col>arima_th.64</th><th scope=col>arima_th.8</th><th scope=col>arima_th.all</th><th scope=col>ericsson</th><th scope=col>ets_th.16</th><th scope=col>ets_th.2</th><th scope=col>ets_th.32</th><th scope=col>ets_th.4</th><th scope=col>ets_th.64</th><th scope=col>ets_th.8</th><th scope=col>ets_th.all</th><th scope=col>naive</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><th scope=row>kernel_networking</th><td>  2.14</td><td>  3.27</td><td>  2.44</td><td>  3.03</td><td>  2.17</td><td>  2.72</td><td>  1.95</td><td>  2.53</td><td>  2.16</td><td>  2.55</td><td>  2.21</td><td>  2.66</td><td>  2.14</td><td>  2.12</td><td>  1.93</td><td>  1.83</td></tr>\n",
       "\t<tr><th scope=row>eclipse_platform</th><td> 49.72</td><td> 76.68</td><td> 48.98</td><td> 69.90</td><td> 45.48</td><td> 51.66</td><td> 45.43</td><td> 46.65</td><td> 48.46</td><td> 54.50</td><td> 47.20</td><td> 53.16</td><td> 45.40</td><td> 53.16</td><td> 44.05</td><td> 49.11</td></tr>\n",
       "\t<tr><th scope=row>eclipse_data_tools</th><td>  8.91</td><td> 10.84</td><td>  8.89</td><td> 11.67</td><td>  8.49</td><td>  9.94</td><td>  8.12</td><td>  8.52</td><td>  8.78</td><td>  9.63</td><td>  8.68</td><td>  9.59</td><td>  8.47</td><td>  9.32</td><td>  8.37</td><td>  8.75</td></tr>\n",
       "\t<tr><th scope=row>eclipse_birt</th><td> 22.98</td><td> 38.44</td><td> 22.76</td><td> 32.32</td><td> 22.63</td><td> 28.66</td><td> 23.51</td><td> 23.61</td><td> 24.72</td><td> 26.89</td><td> 25.58</td><td> 27.11</td><td> 22.34</td><td> 24.98</td><td> 20.67</td><td> 24.15</td></tr>\n",
       "\t<tr><th scope=row>eclipse_jdt</th><td> 25.20</td><td> 37.60</td><td> 21.41</td><td> 39.50</td><td> 20.09</td><td> 31.70</td><td> 24.57</td><td> 26.01</td><td> 24.60</td><td> 27.27</td><td> 20.85</td><td> 30.22</td><td> 19.68</td><td> 28.88</td><td> 22.17</td><td> 23.54</td></tr>\n",
       "\t<tr><th scope=row>eclipse_pde</th><td>  9.21</td><td> 11.51</td><td> 10.17</td><td> 11.47</td><td>  9.38</td><td>  9.73</td><td>  8.07</td><td>  9.67</td><td>  8.42</td><td>  9.11</td><td>  9.08</td><td>  9.46</td><td>  9.47</td><td>  9.35</td><td>  8.24</td><td>  8.35</td></tr>\n",
       "\t<tr><th scope=row>mozilla_calendar</th><td> 19.09</td><td> 27.10</td><td> 20.03</td><td> 23.51</td><td> 18.28</td><td> 19.94</td><td> 18.97</td><td> 14.86</td><td> 16.65</td><td> 19.12</td><td> 14.62</td><td> 21.14</td><td> 14.79</td><td> 18.13</td><td> 13.75</td><td> 14.85</td></tr>\n",
       "\t<tr><th scope=row>mozilla_firefox</th><td> 79.41</td><td>141.15</td><td> 62.65</td><td>112.73</td><td> 63.15</td><td> 82.05</td><td> 60.17</td><td> 71.03</td><td> 79.00</td><td> 86.12</td><td> 67.09</td><td> 85.12</td><td> 60.69</td><td> 89.07</td><td> 58.61</td><td> 70.12</td></tr>\n",
       "\t<tr><th scope=row>mozilla_core</th><td>152.18</td><td>263.84</td><td>146.26</td><td>210.33</td><td>127.63</td><td>174.31</td><td>117.35</td><td>131.98</td><td>148.97</td><td>161.08</td><td>139.83</td><td>162.24</td><td>139.33</td><td>171.80</td><td>125.02</td><td>144.47</td></tr>\n",
       "\t<tr><th scope=row>mozilla_thunderbird</th><td> 23.08</td><td> 42.60</td><td> 25.15</td><td> 32.04</td><td> 21.68</td><td> 24.46</td><td> 18.70</td><td> 20.94</td><td> 24.09</td><td> 25.57</td><td> 20.89</td><td> 26.69</td><td> 20.73</td><td> 25.04</td><td> 19.52</td><td> 22.04</td></tr>\n",
       "\t<tr><th scope=row>kernel_IO_Storage</th><td>  3.19</td><td>  3.91</td><td>  3.26</td><td>  3.88</td><td>  3.69</td><td>  3.25</td><td>  2.80</td><td>  3.19</td><td>  2.93</td><td>  3.02</td><td>  2.88</td><td>  3.08</td><td>  2.84</td><td>  2.80</td><td>  2.77</td><td>  2.72</td></tr>\n",
       "\t<tr><th scope=row>kernel_file_system</th><td>  4.91</td><td>  5.99</td><td>  4.89</td><td>  6.66</td><td>  3.12</td><td>  4.94</td><td>  2.68</td><td>  3.50</td><td>  4.67</td><td>  5.51</td><td>  4.13</td><td>  5.01</td><td>  3.54</td><td>  4.68</td><td>  2.74</td><td>  2.83</td></tr>\n",
       "\t<tr><th scope=row>open_office_writer</th><td> 85.51</td><td>101.56</td><td> 36.85</td><td> 89.02</td><td> 43.83</td><td> 81.48</td><td> 46.25</td><td> 54.39</td><td> 78.91</td><td> 85.88</td><td> 41.20</td><td> 85.89</td><td> 43.57</td><td> 79.84</td><td> 44.70</td><td> 46.26</td></tr>\n",
       "\t<tr><th scope=row>open_office_calc</th><td> 27.11</td><td> 36.14</td><td> 19.82</td><td> 30.37</td><td>  9.56</td><td> 26.07</td><td> 17.21</td><td> 19.18</td><td>  9.53</td><td> 27.59</td><td>  9.29</td><td> 26.05</td><td>  9.62</td><td> 26.10</td><td> 11.70</td><td> 13.84</td></tr>\n",
       "\t<tr><th scope=row>open_office_draw</th><td>  8.64</td><td>  9.44</td><td>  7.18</td><td>  9.84</td><td>  3.93</td><td>  9.15</td><td>  6.24</td><td>  4.96</td><td>  6.36</td><td>  7.59</td><td>  8.58</td><td>  6.22</td><td>  4.51</td><td>  6.86</td><td>  3.84</td><td>  2.71</td></tr>\n",
       "\t<tr><th scope=row>apache_ant</th><td> 20.61</td><td> 28.87</td><td> 20.51</td><td> 22.32</td><td> 20.66</td><td> 20.17</td><td> 20.12</td><td> 19.79</td><td> 19.93</td><td> 19.96</td><td> 20.07</td><td> 20.13</td><td> 20.68</td><td> 20.13</td><td> 20.43</td><td> 22.69</td></tr>\n",
       "\t<tr><th scope=row>apache_apache2</th><td>  6.54</td><td> 12.47</td><td>  6.89</td><td> 12.23</td><td>  6.52</td><td>  7.23</td><td>  6.13</td><td> 10.49</td><td>  6.32</td><td>  7.33</td><td>  6.25</td><td>  7.08</td><td>  6.40</td><td>  6.86</td><td>  7.36</td><td>  7.71</td></tr>\n",
       "\t<tr><th scope=row>libre_office_writer</th><td> 26.64</td><td> 54.40</td><td> 24.48</td><td> 37.51</td><td> 21.91</td><td> 27.13</td><td> 20.53</td><td> 24.19</td><td> 22.79</td><td> 25.96</td><td> 23.42</td><td> 23.81</td><td> 21.35</td><td> 27.72</td><td> 20.90</td><td> 28.79</td></tr>\n",
       "\t<tr><th scope=row>libre_office_calc</th><td> 14.39</td><td> 32.06</td><td> 14.18</td><td> 20.33</td><td> 13.87</td><td> 12.87</td><td> 11.38</td><td> 12.69</td><td> 15.30</td><td> 16.01</td><td> 14.90</td><td> 14.62</td><td> 13.52</td><td> 16.42</td><td> 12.01</td><td> 17.62</td></tr>\n",
       "\t<tr><th scope=row>libre_office_draw</th><td>  3.57</td><td>  6.42</td><td>  3.81</td><td>  5.27</td><td>  3.37</td><td>  4.06</td><td>  3.46</td><td>  3.43</td><td>  3.74</td><td>  4.16</td><td>  3.72</td><td>  4.21</td><td>  3.61</td><td>  4.23</td><td>  3.51</td><td>  3.91</td></tr>\n",
       "\t<tr><th scope=row>mean_error</th><td> 29.65</td><td> 47.21</td><td> 25.53</td><td> 39.20</td><td> 23.47</td><td> 31.58</td><td> 23.18</td><td> 25.58</td><td> 27.82</td><td> 31.24</td><td> 24.52</td><td> 31.17</td><td> 23.63</td><td> 31.37</td><td> 22.61</td><td> 25.81</td></tr>\n",
       "\t<tr><th scope=row>errors_standard_deviation</th><td> 37.09</td><td> 62.12</td><td> 32.44</td><td> 49.77</td><td> 29.39</td><td> 40.83</td><td> 27.34</td><td> 30.99</td><td> 36.36</td><td> 39.14</td><td> 31.84</td><td> 39.29</td><td> 31.49</td><td> 40.92</td><td> 28.69</td><td> 33.21</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllllllllllll}\n",
       "  & arima\\_th.16 & arima\\_th.2 & arima\\_th.32 & arima\\_th.4 & arima\\_th.64 & arima\\_th.8 & arima\\_th.all & ericsson & ets\\_th.16 & ets\\_th.2 & ets\\_th.32 & ets\\_th.4 & ets\\_th.64 & ets\\_th.8 & ets\\_th.all & naive\\\\\n",
       "\\hline\n",
       "\tkernel\\_networking &   2.14 &   3.27 &   2.44 &   3.03 &   2.17 &   2.72 &   1.95 &   2.53 &   2.16 &   2.55 &   2.21 &   2.66 &   2.14 &   2.12 &   1.93 &   1.83\\\\\n",
       "\teclipse\\_platform &  49.72 &  76.68 &  48.98 &  69.90 &  45.48 &  51.66 &  45.43 &  46.65 &  48.46 &  54.50 &  47.20 &  53.16 &  45.40 &  53.16 &  44.05 &  49.11\\\\\n",
       "\teclipse\\_data\\_tools &   8.91 &  10.84 &   8.89 &  11.67 &   8.49 &   9.94 &   8.12 &   8.52 &   8.78 &   9.63 &   8.68 &   9.59 &   8.47 &   9.32 &   8.37 &   8.75\\\\\n",
       "\teclipse\\_birt &  22.98 &  38.44 &  22.76 &  32.32 &  22.63 &  28.66 &  23.51 &  23.61 &  24.72 &  26.89 &  25.58 &  27.11 &  22.34 &  24.98 &  20.67 &  24.15\\\\\n",
       "\teclipse\\_jdt &  25.20 &  37.60 &  21.41 &  39.50 &  20.09 &  31.70 &  24.57 &  26.01 &  24.60 &  27.27 &  20.85 &  30.22 &  19.68 &  28.88 &  22.17 &  23.54\\\\\n",
       "\teclipse\\_pde &   9.21 &  11.51 &  10.17 &  11.47 &   9.38 &   9.73 &   8.07 &   9.67 &   8.42 &   9.11 &   9.08 &   9.46 &   9.47 &   9.35 &   8.24 &   8.35\\\\\n",
       "\tmozilla\\_calendar &  19.09 &  27.10 &  20.03 &  23.51 &  18.28 &  19.94 &  18.97 &  14.86 &  16.65 &  19.12 &  14.62 &  21.14 &  14.79 &  18.13 &  13.75 &  14.85\\\\\n",
       "\tmozilla\\_firefox &  79.41 & 141.15 &  62.65 & 112.73 &  63.15 &  82.05 &  60.17 &  71.03 &  79.00 &  86.12 &  67.09 &  85.12 &  60.69 &  89.07 &  58.61 &  70.12\\\\\n",
       "\tmozilla\\_core & 152.18 & 263.84 & 146.26 & 210.33 & 127.63 & 174.31 & 117.35 & 131.98 & 148.97 & 161.08 & 139.83 & 162.24 & 139.33 & 171.80 & 125.02 & 144.47\\\\\n",
       "\tmozilla\\_thunderbird &  23.08 &  42.60 &  25.15 &  32.04 &  21.68 &  24.46 &  18.70 &  20.94 &  24.09 &  25.57 &  20.89 &  26.69 &  20.73 &  25.04 &  19.52 &  22.04\\\\\n",
       "\tkernel\\_IO\\_Storage &   3.19 &   3.91 &   3.26 &   3.88 &   3.69 &   3.25 &   2.80 &   3.19 &   2.93 &   3.02 &   2.88 &   3.08 &   2.84 &   2.80 &   2.77 &   2.72\\\\\n",
       "\tkernel\\_file\\_system &   4.91 &   5.99 &   4.89 &   6.66 &   3.12 &   4.94 &   2.68 &   3.50 &   4.67 &   5.51 &   4.13 &   5.01 &   3.54 &   4.68 &   2.74 &   2.83\\\\\n",
       "\topen\\_office\\_writer &  85.51 & 101.56 &  36.85 &  89.02 &  43.83 &  81.48 &  46.25 &  54.39 &  78.91 &  85.88 &  41.20 &  85.89 &  43.57 &  79.84 &  44.70 &  46.26\\\\\n",
       "\topen\\_office\\_calc &  27.11 &  36.14 &  19.82 &  30.37 &   9.56 &  26.07 &  17.21 &  19.18 &   9.53 &  27.59 &   9.29 &  26.05 &   9.62 &  26.10 &  11.70 &  13.84\\\\\n",
       "\topen\\_office\\_draw &   8.64 &   9.44 &   7.18 &   9.84 &   3.93 &   9.15 &   6.24 &   4.96 &   6.36 &   7.59 &   8.58 &   6.22 &   4.51 &   6.86 &   3.84 &   2.71\\\\\n",
       "\tapache\\_ant &  20.61 &  28.87 &  20.51 &  22.32 &  20.66 &  20.17 &  20.12 &  19.79 &  19.93 &  19.96 &  20.07 &  20.13 &  20.68 &  20.13 &  20.43 &  22.69\\\\\n",
       "\tapache\\_apache2 &   6.54 &  12.47 &   6.89 &  12.23 &   6.52 &   7.23 &   6.13 &  10.49 &   6.32 &   7.33 &   6.25 &   7.08 &   6.40 &   6.86 &   7.36 &   7.71\\\\\n",
       "\tlibre\\_office\\_writer &  26.64 &  54.40 &  24.48 &  37.51 &  21.91 &  27.13 &  20.53 &  24.19 &  22.79 &  25.96 &  23.42 &  23.81 &  21.35 &  27.72 &  20.90 &  28.79\\\\\n",
       "\tlibre\\_office\\_calc &  14.39 &  32.06 &  14.18 &  20.33 &  13.87 &  12.87 &  11.38 &  12.69 &  15.30 &  16.01 &  14.90 &  14.62 &  13.52 &  16.42 &  12.01 &  17.62\\\\\n",
       "\tlibre\\_office\\_draw &   3.57 &   6.42 &   3.81 &   5.27 &   3.37 &   4.06 &   3.46 &   3.43 &   3.74 &   4.16 &   3.72 &   4.21 &   3.61 &   4.23 &   3.51 &   3.91\\\\\n",
       "\tmean\\_error &  29.65 &  47.21 &  25.53 &  39.20 &  23.47 &  31.58 &  23.18 &  25.58 &  27.82 &  31.24 &  24.52 &  31.17 &  23.63 &  31.37 &  22.61 &  25.81\\\\\n",
       "\terrors\\_standard\\_deviation &  37.09 &  62.12 &  32.44 &  49.77 &  29.39 &  40.83 &  27.34 &  30.99 &  36.36 &  39.14 &  31.84 &  39.29 &  31.49 &  40.92 &  28.69 &  33.21\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| <!--/--> | arima_th.16 | arima_th.2 | arima_th.32 | arima_th.4 | arima_th.64 | arima_th.8 | arima_th.all | ericsson | ets_th.16 | ets_th.2 | ets_th.32 | ets_th.4 | ets_th.64 | ets_th.8 | ets_th.all | naive | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking |   2.14 |   3.27 |   2.44 |   3.03 |   2.17 |   2.72 |   1.95 |   2.53 |   2.16 |   2.55 |   2.21 |   2.66 |   2.14 |   2.12 |   1.93 |   1.83 | \n",
       "| eclipse_platform |  49.72 |  76.68 |  48.98 |  69.90 |  45.48 |  51.66 |  45.43 |  46.65 |  48.46 |  54.50 |  47.20 |  53.16 |  45.40 |  53.16 |  44.05 |  49.11 | \n",
       "| eclipse_data_tools |   8.91 |  10.84 |   8.89 |  11.67 |   8.49 |   9.94 |   8.12 |   8.52 |   8.78 |   9.63 |   8.68 |   9.59 |   8.47 |   9.32 |   8.37 |   8.75 | \n",
       "| eclipse_birt |  22.98 |  38.44 |  22.76 |  32.32 |  22.63 |  28.66 |  23.51 |  23.61 |  24.72 |  26.89 |  25.58 |  27.11 |  22.34 |  24.98 |  20.67 |  24.15 | \n",
       "| eclipse_jdt |  25.20 |  37.60 |  21.41 |  39.50 |  20.09 |  31.70 |  24.57 |  26.01 |  24.60 |  27.27 |  20.85 |  30.22 |  19.68 |  28.88 |  22.17 |  23.54 | \n",
       "| eclipse_pde |   9.21 |  11.51 |  10.17 |  11.47 |   9.38 |   9.73 |   8.07 |   9.67 |   8.42 |   9.11 |   9.08 |   9.46 |   9.47 |   9.35 |   8.24 |   8.35 | \n",
       "| mozilla_calendar |  19.09 |  27.10 |  20.03 |  23.51 |  18.28 |  19.94 |  18.97 |  14.86 |  16.65 |  19.12 |  14.62 |  21.14 |  14.79 |  18.13 |  13.75 |  14.85 | \n",
       "| mozilla_firefox |  79.41 | 141.15 |  62.65 | 112.73 |  63.15 |  82.05 |  60.17 |  71.03 |  79.00 |  86.12 |  67.09 |  85.12 |  60.69 |  89.07 |  58.61 |  70.12 | \n",
       "| mozilla_core | 152.18 | 263.84 | 146.26 | 210.33 | 127.63 | 174.31 | 117.35 | 131.98 | 148.97 | 161.08 | 139.83 | 162.24 | 139.33 | 171.80 | 125.02 | 144.47 | \n",
       "| mozilla_thunderbird |  23.08 |  42.60 |  25.15 |  32.04 |  21.68 |  24.46 |  18.70 |  20.94 |  24.09 |  25.57 |  20.89 |  26.69 |  20.73 |  25.04 |  19.52 |  22.04 | \n",
       "| kernel_IO_Storage |   3.19 |   3.91 |   3.26 |   3.88 |   3.69 |   3.25 |   2.80 |   3.19 |   2.93 |   3.02 |   2.88 |   3.08 |   2.84 |   2.80 |   2.77 |   2.72 | \n",
       "| kernel_file_system |   4.91 |   5.99 |   4.89 |   6.66 |   3.12 |   4.94 |   2.68 |   3.50 |   4.67 |   5.51 |   4.13 |   5.01 |   3.54 |   4.68 |   2.74 |   2.83 | \n",
       "| open_office_writer |  85.51 | 101.56 |  36.85 |  89.02 |  43.83 |  81.48 |  46.25 |  54.39 |  78.91 |  85.88 |  41.20 |  85.89 |  43.57 |  79.84 |  44.70 |  46.26 | \n",
       "| open_office_calc |  27.11 |  36.14 |  19.82 |  30.37 |   9.56 |  26.07 |  17.21 |  19.18 |   9.53 |  27.59 |   9.29 |  26.05 |   9.62 |  26.10 |  11.70 |  13.84 | \n",
       "| open_office_draw |   8.64 |   9.44 |   7.18 |   9.84 |   3.93 |   9.15 |   6.24 |   4.96 |   6.36 |   7.59 |   8.58 |   6.22 |   4.51 |   6.86 |   3.84 |   2.71 | \n",
       "| apache_ant |  20.61 |  28.87 |  20.51 |  22.32 |  20.66 |  20.17 |  20.12 |  19.79 |  19.93 |  19.96 |  20.07 |  20.13 |  20.68 |  20.13 |  20.43 |  22.69 | \n",
       "| apache_apache2 |   6.54 |  12.47 |   6.89 |  12.23 |   6.52 |   7.23 |   6.13 |  10.49 |   6.32 |   7.33 |   6.25 |   7.08 |   6.40 |   6.86 |   7.36 |   7.71 | \n",
       "| libre_office_writer |  26.64 |  54.40 |  24.48 |  37.51 |  21.91 |  27.13 |  20.53 |  24.19 |  22.79 |  25.96 |  23.42 |  23.81 |  21.35 |  27.72 |  20.90 |  28.79 | \n",
       "| libre_office_calc |  14.39 |  32.06 |  14.18 |  20.33 |  13.87 |  12.87 |  11.38 |  12.69 |  15.30 |  16.01 |  14.90 |  14.62 |  13.52 |  16.42 |  12.01 |  17.62 | \n",
       "| libre_office_draw |   3.57 |   6.42 |   3.81 |   5.27 |   3.37 |   4.06 |   3.46 |   3.43 |   3.74 |   4.16 |   3.72 |   4.21 |   3.61 |   4.23 |   3.51 |   3.91 | \n",
       "| mean_error |  29.65 |  47.21 |  25.53 |  39.20 |  23.47 |  31.58 |  23.18 |  25.58 |  27.82 |  31.24 |  24.52 |  31.17 |  23.63 |  31.37 |  22.61 |  25.81 | \n",
       "| errors_standard_deviation |  37.09 |  62.12 |  32.44 |  49.77 |  29.39 |  40.83 |  27.34 |  30.99 |  36.36 |  39.14 |  31.84 |  39.29 |  31.49 |  40.92 |  28.69 |  33.21 | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "                          arima_th.16 arima_th.2 arima_th.32 arima_th.4\n",
       "kernel_networking           2.14        3.27       2.44        3.03    \n",
       "eclipse_platform           49.72       76.68      48.98       69.90    \n",
       "eclipse_data_tools          8.91       10.84       8.89       11.67    \n",
       "eclipse_birt               22.98       38.44      22.76       32.32    \n",
       "eclipse_jdt                25.20       37.60      21.41       39.50    \n",
       "eclipse_pde                 9.21       11.51      10.17       11.47    \n",
       "mozilla_calendar           19.09       27.10      20.03       23.51    \n",
       "mozilla_firefox            79.41      141.15      62.65      112.73    \n",
       "mozilla_core              152.18      263.84     146.26      210.33    \n",
       "mozilla_thunderbird        23.08       42.60      25.15       32.04    \n",
       "kernel_IO_Storage           3.19        3.91       3.26        3.88    \n",
       "kernel_file_system          4.91        5.99       4.89        6.66    \n",
       "open_office_writer         85.51      101.56      36.85       89.02    \n",
       "open_office_calc           27.11       36.14      19.82       30.37    \n",
       "open_office_draw            8.64        9.44       7.18        9.84    \n",
       "apache_ant                 20.61       28.87      20.51       22.32    \n",
       "apache_apache2              6.54       12.47       6.89       12.23    \n",
       "libre_office_writer        26.64       54.40      24.48       37.51    \n",
       "libre_office_calc          14.39       32.06      14.18       20.33    \n",
       "libre_office_draw           3.57        6.42       3.81        5.27    \n",
       "mean_error                 29.65       47.21      25.53       39.20    \n",
       "errors_standard_deviation  37.09       62.12      32.44       49.77    \n",
       "                          arima_th.64 arima_th.8 arima_th.all ericsson\n",
       "kernel_networking           2.17        2.72       1.95         2.53  \n",
       "eclipse_platform           45.48       51.66      45.43        46.65  \n",
       "eclipse_data_tools          8.49        9.94       8.12         8.52  \n",
       "eclipse_birt               22.63       28.66      23.51        23.61  \n",
       "eclipse_jdt                20.09       31.70      24.57        26.01  \n",
       "eclipse_pde                 9.38        9.73       8.07         9.67  \n",
       "mozilla_calendar           18.28       19.94      18.97        14.86  \n",
       "mozilla_firefox            63.15       82.05      60.17        71.03  \n",
       "mozilla_core              127.63      174.31     117.35       131.98  \n",
       "mozilla_thunderbird        21.68       24.46      18.70        20.94  \n",
       "kernel_IO_Storage           3.69        3.25       2.80         3.19  \n",
       "kernel_file_system          3.12        4.94       2.68         3.50  \n",
       "open_office_writer         43.83       81.48      46.25        54.39  \n",
       "open_office_calc            9.56       26.07      17.21        19.18  \n",
       "open_office_draw            3.93        9.15       6.24         4.96  \n",
       "apache_ant                 20.66       20.17      20.12        19.79  \n",
       "apache_apache2              6.52        7.23       6.13        10.49  \n",
       "libre_office_writer        21.91       27.13      20.53        24.19  \n",
       "libre_office_calc          13.87       12.87      11.38        12.69  \n",
       "libre_office_draw           3.37        4.06       3.46         3.43  \n",
       "mean_error                 23.47       31.58      23.18        25.58  \n",
       "errors_standard_deviation  29.39       40.83      27.34        30.99  \n",
       "                          ets_th.16 ets_th.2 ets_th.32 ets_th.4 ets_th.64\n",
       "kernel_networking           2.16      2.55     2.21      2.66     2.14   \n",
       "eclipse_platform           48.46     54.50    47.20     53.16    45.40   \n",
       "eclipse_data_tools          8.78      9.63     8.68      9.59     8.47   \n",
       "eclipse_birt               24.72     26.89    25.58     27.11    22.34   \n",
       "eclipse_jdt                24.60     27.27    20.85     30.22    19.68   \n",
       "eclipse_pde                 8.42      9.11     9.08      9.46     9.47   \n",
       "mozilla_calendar           16.65     19.12    14.62     21.14    14.79   \n",
       "mozilla_firefox            79.00     86.12    67.09     85.12    60.69   \n",
       "mozilla_core              148.97    161.08   139.83    162.24   139.33   \n",
       "mozilla_thunderbird        24.09     25.57    20.89     26.69    20.73   \n",
       "kernel_IO_Storage           2.93      3.02     2.88      3.08     2.84   \n",
       "kernel_file_system          4.67      5.51     4.13      5.01     3.54   \n",
       "open_office_writer         78.91     85.88    41.20     85.89    43.57   \n",
       "open_office_calc            9.53     27.59     9.29     26.05     9.62   \n",
       "open_office_draw            6.36      7.59     8.58      6.22     4.51   \n",
       "apache_ant                 19.93     19.96    20.07     20.13    20.68   \n",
       "apache_apache2              6.32      7.33     6.25      7.08     6.40   \n",
       "libre_office_writer        22.79     25.96    23.42     23.81    21.35   \n",
       "libre_office_calc          15.30     16.01    14.90     14.62    13.52   \n",
       "libre_office_draw           3.74      4.16     3.72      4.21     3.61   \n",
       "mean_error                 27.82     31.24    24.52     31.17    23.63   \n",
       "errors_standard_deviation  36.36     39.14    31.84     39.29    31.49   \n",
       "                          ets_th.8 ets_th.all naive \n",
       "kernel_networking           2.12     1.93       1.83\n",
       "eclipse_platform           53.16    44.05      49.11\n",
       "eclipse_data_tools          9.32     8.37       8.75\n",
       "eclipse_birt               24.98    20.67      24.15\n",
       "eclipse_jdt                28.88    22.17      23.54\n",
       "eclipse_pde                 9.35     8.24       8.35\n",
       "mozilla_calendar           18.13    13.75      14.85\n",
       "mozilla_firefox            89.07    58.61      70.12\n",
       "mozilla_core              171.80   125.02     144.47\n",
       "mozilla_thunderbird        25.04    19.52      22.04\n",
       "kernel_IO_Storage           2.80     2.77       2.72\n",
       "kernel_file_system          4.68     2.74       2.83\n",
       "open_office_writer         79.84    44.70      46.26\n",
       "open_office_calc           26.10    11.70      13.84\n",
       "open_office_draw            6.86     3.84       2.71\n",
       "apache_ant                 20.13    20.43      22.69\n",
       "apache_apache2              6.86     7.36       7.71\n",
       "libre_office_writer        27.72    20.90      28.79\n",
       "libre_office_calc          16.42    12.01      17.62\n",
       "libre_office_draw           4.23     3.51       3.91\n",
       "mean_error                 31.37    22.61      25.81\n",
       "errors_standard_deviation  40.92    28.69      33.21"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "errors_evaluation = read.csv(file=paste(\"data/errors_evaluation_week_1.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "errors_evaluation\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backlogs details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Products</th><th scope=col>Acf_lag_1</th><th scope=col>Acf_lag_2</th><th scope=col>Acf_lag_3</th><th scope=col>Acf_lag_4</th><th scope=col>Acf_lag_5</th><th scope=col>Acf_lag_6</th><th scope=col>Acf_lag_7</th><th scope=col>Acf_lag_8</th><th scope=col>Acf_lag_9</th><th scope=col>⋯</th><th scope=col>Acf_lag_23</th><th scope=col>Acf_lag_24</th><th scope=col>Acf_lag_25</th><th scope=col>Acf_lag_26</th><th scope=col>Acf_lag_27</th><th scope=col>Acf_lag_28</th><th scope=col>Acf_lag_29</th><th scope=col>Acf_lag_30</th><th scope=col>Mean_backlog_size</th><th scope=col>Mean_backlog_size_change</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>kernel_networking  </td><td>1                  </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.95               </td><td>⋯                  </td><td>0.83               </td><td>0.82               </td><td>0.81               </td><td>0.80               </td><td>0.79               </td><td>0.78               </td><td>0.78               </td><td>0.77               </td><td>  112.38           </td><td>  2.29             </td></tr>\n",
       "\t<tr><td>eclipse_platform   </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>⋯                  </td><td>0.90               </td><td>0.90               </td><td>0.89               </td><td>0.89               </td><td>0.88               </td><td>0.88               </td><td>0.87               </td><td>0.87               </td><td> 7240.95           </td><td> 47.60             </td></tr>\n",
       "\t<tr><td>eclipse_data_tools </td><td>1                  </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.94               </td><td>0.92               </td><td>0.91               </td><td>⋯                  </td><td>0.77               </td><td>0.76               </td><td>0.75               </td><td>0.74               </td><td>0.73               </td><td>0.72               </td><td>0.71               </td><td>0.70               </td><td>  159.05           </td><td>  4.54             </td></tr>\n",
       "\t<tr><td>eclipse_birt       </td><td>1                  </td><td>1.00               </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>⋯                  </td><td>0.90               </td><td>0.89               </td><td>0.89               </td><td>0.88               </td><td>0.88               </td><td>0.87               </td><td>0.87               </td><td>0.86               </td><td> 1000.21           </td><td> 18.89             </td></tr>\n",
       "\t<tr><td>eclipse_jdt        </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>⋯                  </td><td>0.92               </td><td>0.92               </td><td>0.91               </td><td>0.91               </td><td>0.90               </td><td>0.90               </td><td>0.90               </td><td>0.89               </td><td> 3329.15           </td><td> 28.73             </td></tr>\n",
       "\t<tr><td>eclipse_pde        </td><td>1                  </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>0.96               </td><td>0.96               </td><td>0.95               </td><td>⋯                  </td><td>0.89               </td><td>0.89               </td><td>0.88               </td><td>0.88               </td><td>0.87               </td><td>0.87               </td><td>0.86               </td><td>0.86               </td><td>  850.86           </td><td>  8.27             </td></tr>\n",
       "\t<tr><td>mozilla_calendar   </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.95               </td><td>⋯                  </td><td>0.85               </td><td>0.84               </td><td>0.83               </td><td>0.82               </td><td>0.82               </td><td>0.81               </td><td>0.80               </td><td>0.80               </td><td> 1219.78           </td><td> 12.59             </td></tr>\n",
       "\t<tr><td>mozilla_firefox    </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.98               </td><td>⋯                  </td><td>0.93               </td><td>0.93               </td><td>0.93               </td><td>0.92               </td><td>0.92               </td><td>0.92               </td><td>0.91               </td><td>0.91               </td><td>11354.33           </td><td> 82.95             </td></tr>\n",
       "\t<tr><td>mozilla_core       </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>⋯                  </td><td>0.92               </td><td>0.91               </td><td>0.91               </td><td>0.91               </td><td>0.90               </td><td>0.90               </td><td>0.90               </td><td>0.89               </td><td>29050.79           </td><td>141.83             </td></tr>\n",
       "\t<tr><td>mozilla_thunderbird</td><td>1                  </td><td>1.00               </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.98               </td><td>⋯                  </td><td>0.94               </td><td>0.93               </td><td>0.93               </td><td>0.93               </td><td>0.92               </td><td>0.92               </td><td>0.92               </td><td>0.91               </td><td> 3708.50           </td><td> 22.94             </td></tr>\n",
       "\t<tr><td>kernel_IO_Storage  </td><td>1                  </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.94               </td><td>⋯                  </td><td>0.82               </td><td>0.81               </td><td>0.80               </td><td>0.79               </td><td>0.78               </td><td>0.77               </td><td>0.76               </td><td>0.76               </td><td>  161.42           </td><td>  2.59             </td></tr>\n",
       "\t<tr><td>kernel_file_system </td><td>1                  </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.93               </td><td>0.92               </td><td>0.91               </td><td>⋯                  </td><td>0.74               </td><td>0.73               </td><td>0.71               </td><td>0.70               </td><td>0.69               </td><td>0.68               </td><td>0.67               </td><td>0.65               </td><td>  190.59           </td><td>  3.57             </td></tr>\n",
       "\t<tr><td>open_office_writer </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>0.97               </td><td>⋯                  </td><td>0.90               </td><td>0.90               </td><td>0.89               </td><td>0.89               </td><td>0.88               </td><td>0.88               </td><td>0.87               </td><td>0.87               </td><td> 8123.17           </td><td> 43.62             </td></tr>\n",
       "\t<tr><td>open_office_calc   </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>⋯                  </td><td>0.91               </td><td>0.90               </td><td>0.90               </td><td>0.89               </td><td>0.89               </td><td>0.88               </td><td>0.88               </td><td>0.87               </td><td> 3245.73           </td><td> 18.99             </td></tr>\n",
       "\t<tr><td>open_office_draw   </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>0.96               </td><td>0.96               </td><td>⋯                  </td><td>0.88               </td><td>0.88               </td><td>0.87               </td><td>0.87               </td><td>0.86               </td><td>0.86               </td><td>0.85               </td><td>0.84               </td><td>  810.53           </td><td>  4.97             </td></tr>\n",
       "\t<tr><td>apache_ant         </td><td>1                  </td><td>1.00               </td><td>0.99               </td><td>0.99               </td><td>0.99               </td><td>0.98               </td><td>0.98               </td><td>0.97               </td><td>0.97               </td><td>⋯                  </td><td>0.91               </td><td>0.91               </td><td>0.91               </td><td>0.90               </td><td>0.90               </td><td>0.89               </td><td>0.89               </td><td>0.88               </td><td> 1213.95           </td><td> 10.74             </td></tr>\n",
       "\t<tr><td>apache_apache2     </td><td>1                  </td><td>0.99               </td><td>0.97               </td><td>0.96               </td><td>0.94               </td><td>0.93               </td><td>0.91               </td><td>0.90               </td><td>0.88               </td><td>⋯                  </td><td>0.69               </td><td>0.68               </td><td>0.66               </td><td>0.65               </td><td>0.63               </td><td>0.62               </td><td>0.60               </td><td>0.58               </td><td>  976.48           </td><td>  8.71             </td></tr>\n",
       "\t<tr><td>libre_office_writer</td><td>1                  </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.94               </td><td>0.93               </td><td>0.92               </td><td>⋯                  </td><td>0.79               </td><td>0.78               </td><td>0.78               </td><td>0.77               </td><td>0.76               </td><td>0.75               </td><td>0.74               </td><td>0.73               </td><td> 3245.32           </td><td> 28.38             </td></tr>\n",
       "\t<tr><td>libre_office_calc  </td><td>1                  </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.96               </td><td>0.95               </td><td>0.94               </td><td>0.94               </td><td>0.93               </td><td>⋯                  </td><td>0.79               </td><td>0.78               </td><td>0.77               </td><td>0.76               </td><td>0.75               </td><td>0.75               </td><td>0.74               </td><td>0.73               </td><td> 1816.79           </td><td> 17.94             </td></tr>\n",
       "\t<tr><td>libre_office_draw  </td><td>1                  </td><td>0.99               </td><td>0.98               </td><td>0.97               </td><td>0.96               </td><td>0.94               </td><td>0.93               </td><td>0.92               </td><td>0.91               </td><td>⋯                  </td><td>0.75               </td><td>0.74               </td><td>0.73               </td><td>0.72               </td><td>0.71               </td><td>0.70               </td><td>0.69               </td><td>0.68               </td><td>  370.36           </td><td>  3.96             </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllllllllllllllllllllllllllllll}\n",
       " Products & Acf\\_lag\\_1 & Acf\\_lag\\_2 & Acf\\_lag\\_3 & Acf\\_lag\\_4 & Acf\\_lag\\_5 & Acf\\_lag\\_6 & Acf\\_lag\\_7 & Acf\\_lag\\_8 & Acf\\_lag\\_9 & ⋯ & Acf\\_lag\\_23 & Acf\\_lag\\_24 & Acf\\_lag\\_25 & Acf\\_lag\\_26 & Acf\\_lag\\_27 & Acf\\_lag\\_28 & Acf\\_lag\\_29 & Acf\\_lag\\_30 & Mean\\_backlog\\_size & Mean\\_backlog\\_size\\_change\\\\\n",
       "\\hline\n",
       "\t kernel\\_networking   & 1                     & 0.99                  & 0.99                  & 0.98                  & 0.97                  & 0.97                  & 0.96                  & 0.95                  & 0.95                  & ⋯                     & 0.83                  & 0.82                  & 0.81                  & 0.80                  & 0.79                  & 0.78                  & 0.78                  & 0.77                  &   112.38              &   2.29               \\\\\n",
       "\t eclipse\\_platform    & 1                     & 1.00                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.98                  & 0.97                  & 0.97                  & ⋯                     & 0.90                  & 0.90                  & 0.89                  & 0.89                  & 0.88                  & 0.88                  & 0.87                  & 0.87                  &  7240.95              &  47.60               \\\\\n",
       "\t eclipse\\_data\\_tools  & 1                       & 0.99                    & 0.98                    & 0.97                    & 0.96                    & 0.95                    & 0.94                    & 0.92                    & 0.91                    & ⋯                       & 0.77                    & 0.76                    & 0.75                    & 0.74                    & 0.73                    & 0.72                    & 0.71                    & 0.70                    &   159.05                &   4.54                 \\\\\n",
       "\t eclipse\\_birt        & 1                     & 1.00                  & 1.00                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.98                  & 0.97                  & ⋯                     & 0.90                  & 0.89                  & 0.89                  & 0.88                  & 0.88                  & 0.87                  & 0.87                  & 0.86                  &  1000.21              &  18.89               \\\\\n",
       "\t eclipse\\_jdt         & 1                     & 1.00                  & 0.99                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.98                  & 0.97                  & ⋯                     & 0.92                  & 0.92                  & 0.91                  & 0.91                  & 0.90                  & 0.90                  & 0.90                  & 0.89                  &  3329.15              &  28.73               \\\\\n",
       "\t eclipse\\_pde         & 1                     & 0.99                  & 0.98                  & 0.98                  & 0.97                  & 0.97                  & 0.96                  & 0.96                  & 0.95                  & ⋯                     & 0.89                  & 0.89                  & 0.88                  & 0.88                  & 0.87                  & 0.87                  & 0.86                  & 0.86                  &   850.86              &   8.27               \\\\\n",
       "\t mozilla\\_calendar    & 1                     & 1.00                  & 0.99                  & 0.98                  & 0.98                  & 0.97                  & 0.96                  & 0.95                  & 0.95                  & ⋯                     & 0.85                  & 0.84                  & 0.83                  & 0.82                  & 0.82                  & 0.81                  & 0.80                  & 0.80                  &  1219.78              &  12.59               \\\\\n",
       "\t mozilla\\_firefox     & 1                     & 1.00                  & 0.99                  & 0.99                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.98                  & ⋯                     & 0.93                  & 0.93                  & 0.93                  & 0.92                  & 0.92                  & 0.92                  & 0.91                  & 0.91                  & 11354.33              &  82.95               \\\\\n",
       "\t mozilla\\_core        & 1                     & 1.00                  & 0.99                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.97                  & 0.97                  & ⋯                     & 0.92                  & 0.91                  & 0.91                  & 0.91                  & 0.90                  & 0.90                  & 0.90                  & 0.89                  & 29050.79              & 141.83               \\\\\n",
       "\t mozilla\\_thunderbird & 1                     & 1.00                  & 1.00                  & 0.99                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.98                  & ⋯                     & 0.94                  & 0.93                  & 0.93                  & 0.93                  & 0.92                  & 0.92                  & 0.92                  & 0.91                  &  3708.50              &  22.94               \\\\\n",
       "\t kernel\\_IO\\_Storage   & 1                       & 0.99                    & 0.99                    & 0.98                    & 0.97                    & 0.97                    & 0.96                    & 0.95                    & 0.94                    & ⋯                       & 0.82                    & 0.81                    & 0.80                    & 0.79                    & 0.78                    & 0.77                    & 0.76                    & 0.76                    &   161.42                &   2.59                 \\\\\n",
       "\t kernel\\_file\\_system  & 1                       & 0.99                    & 0.98                    & 0.97                    & 0.96                    & 0.95                    & 0.93                    & 0.92                    & 0.91                    & ⋯                       & 0.74                    & 0.73                    & 0.71                    & 0.70                    & 0.69                    & 0.68                    & 0.67                    & 0.65                    &   190.59                &   3.57                 \\\\\n",
       "\t open\\_office\\_writer  & 1                       & 1.00                    & 0.99                    & 0.99                    & 0.98                    & 0.98                    & 0.97                    & 0.97                    & 0.97                    & ⋯                       & 0.90                    & 0.90                    & 0.89                    & 0.89                    & 0.88                    & 0.88                    & 0.87                    & 0.87                    &  8123.17                &  43.62                 \\\\\n",
       "\t open\\_office\\_calc    & 1                       & 1.00                    & 0.99                    & 0.99                    & 0.98                    & 0.98                    & 0.98                    & 0.97                    & 0.97                    & ⋯                       & 0.91                    & 0.90                    & 0.90                    & 0.89                    & 0.89                    & 0.88                    & 0.88                    & 0.87                    &  3245.73                &  18.99                 \\\\\n",
       "\t open\\_office\\_draw    & 1                       & 1.00                    & 0.99                    & 0.98                    & 0.98                    & 0.97                    & 0.97                    & 0.96                    & 0.96                    & ⋯                       & 0.88                    & 0.88                    & 0.87                    & 0.87                    & 0.86                    & 0.86                    & 0.85                    & 0.84                    &   810.53                &   4.97                 \\\\\n",
       "\t apache\\_ant          & 1                     & 1.00                  & 0.99                  & 0.99                  & 0.99                  & 0.98                  & 0.98                  & 0.97                  & 0.97                  & ⋯                     & 0.91                  & 0.91                  & 0.91                  & 0.90                  & 0.90                  & 0.89                  & 0.89                  & 0.88                  &  1213.95              &  10.74               \\\\\n",
       "\t apache\\_apache2      & 1                     & 0.99                  & 0.97                  & 0.96                  & 0.94                  & 0.93                  & 0.91                  & 0.90                  & 0.88                  & ⋯                     & 0.69                  & 0.68                  & 0.66                  & 0.65                  & 0.63                  & 0.62                  & 0.60                  & 0.58                  &   976.48              &   8.71               \\\\\n",
       "\t libre\\_office\\_writer & 1                       & 0.99                    & 0.98                    & 0.97                    & 0.96                    & 0.95                    & 0.94                    & 0.93                    & 0.92                    & ⋯                       & 0.79                    & 0.78                    & 0.78                    & 0.77                    & 0.76                    & 0.75                    & 0.74                    & 0.73                    &  3245.32                &  28.38                 \\\\\n",
       "\t libre\\_office\\_calc   & 1                       & 0.99                    & 0.98                    & 0.97                    & 0.96                    & 0.95                    & 0.94                    & 0.94                    & 0.93                    & ⋯                       & 0.79                    & 0.78                    & 0.77                    & 0.76                    & 0.75                    & 0.75                    & 0.74                    & 0.73                    &  1816.79                &  17.94                 \\\\\n",
       "\t libre\\_office\\_draw   & 1                       & 0.99                    & 0.98                    & 0.97                    & 0.96                    & 0.94                    & 0.93                    & 0.92                    & 0.91                    & ⋯                       & 0.75                    & 0.74                    & 0.73                    & 0.72                    & 0.71                    & 0.70                    & 0.69                    & 0.68                    &   370.36                &   3.96                 \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Products | Acf_lag_1 | Acf_lag_2 | Acf_lag_3 | Acf_lag_4 | Acf_lag_5 | Acf_lag_6 | Acf_lag_7 | Acf_lag_8 | Acf_lag_9 | ⋯ | Acf_lag_23 | Acf_lag_24 | Acf_lag_25 | Acf_lag_26 | Acf_lag_27 | Acf_lag_28 | Acf_lag_29 | Acf_lag_30 | Mean_backlog_size | Mean_backlog_size_change | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking   | 1                   | 0.99                | 0.99                | 0.98                | 0.97                | 0.97                | 0.96                | 0.95                | 0.95                | ⋯                   | 0.83                | 0.82                | 0.81                | 0.80                | 0.79                | 0.78                | 0.78                | 0.77                |   112.38            |   2.29              | \n",
       "| eclipse_platform    | 1                   | 1.00                | 0.99                | 0.99                | 0.98                | 0.98                | 0.98                | 0.97                | 0.97                | ⋯                   | 0.90                | 0.90                | 0.89                | 0.89                | 0.88                | 0.88                | 0.87                | 0.87                |  7240.95            |  47.60              | \n",
       "| eclipse_data_tools  | 1                   | 0.99                | 0.98                | 0.97                | 0.96                | 0.95                | 0.94                | 0.92                | 0.91                | ⋯                   | 0.77                | 0.76                | 0.75                | 0.74                | 0.73                | 0.72                | 0.71                | 0.70                |   159.05            |   4.54              | \n",
       "| eclipse_birt        | 1                   | 1.00                | 1.00                | 0.99                | 0.99                | 0.98                | 0.98                | 0.98                | 0.97                | ⋯                   | 0.90                | 0.89                | 0.89                | 0.88                | 0.88                | 0.87                | 0.87                | 0.86                |  1000.21            |  18.89              | \n",
       "| eclipse_jdt         | 1                   | 1.00                | 0.99                | 0.99                | 0.99                | 0.98                | 0.98                | 0.98                | 0.97                | ⋯                   | 0.92                | 0.92                | 0.91                | 0.91                | 0.90                | 0.90                | 0.90                | 0.89                |  3329.15            |  28.73              | \n",
       "| eclipse_pde         | 1                   | 0.99                | 0.98                | 0.98                | 0.97                | 0.97                | 0.96                | 0.96                | 0.95                | ⋯                   | 0.89                | 0.89                | 0.88                | 0.88                | 0.87                | 0.87                | 0.86                | 0.86                |   850.86            |   8.27              | \n",
       "| mozilla_calendar    | 1                   | 1.00                | 0.99                | 0.98                | 0.98                | 0.97                | 0.96                | 0.95                | 0.95                | ⋯                   | 0.85                | 0.84                | 0.83                | 0.82                | 0.82                | 0.81                | 0.80                | 0.80                |  1219.78            |  12.59              | \n",
       "| mozilla_firefox     | 1                   | 1.00                | 0.99                | 0.99                | 0.99                | 0.99                | 0.98                | 0.98                | 0.98                | ⋯                   | 0.93                | 0.93                | 0.93                | 0.92                | 0.92                | 0.92                | 0.91                | 0.91                | 11354.33            |  82.95              | \n",
       "| mozilla_core        | 1                   | 1.00                | 0.99                | 0.99                | 0.99                | 0.98                | 0.98                | 0.97                | 0.97                | ⋯                   | 0.92                | 0.91                | 0.91                | 0.91                | 0.90                | 0.90                | 0.90                | 0.89                | 29050.79            | 141.83              | \n",
       "| mozilla_thunderbird | 1                   | 1.00                | 1.00                | 0.99                | 0.99                | 0.99                | 0.98                | 0.98                | 0.98                | ⋯                   | 0.94                | 0.93                | 0.93                | 0.93                | 0.92                | 0.92                | 0.92                | 0.91                |  3708.50            |  22.94              | \n",
       "| kernel_IO_Storage   | 1                   | 0.99                | 0.99                | 0.98                | 0.97                | 0.97                | 0.96                | 0.95                | 0.94                | ⋯                   | 0.82                | 0.81                | 0.80                | 0.79                | 0.78                | 0.77                | 0.76                | 0.76                |   161.42            |   2.59              | \n",
       "| kernel_file_system  | 1                   | 0.99                | 0.98                | 0.97                | 0.96                | 0.95                | 0.93                | 0.92                | 0.91                | ⋯                   | 0.74                | 0.73                | 0.71                | 0.70                | 0.69                | 0.68                | 0.67                | 0.65                |   190.59            |   3.57              | \n",
       "| open_office_writer  | 1                   | 1.00                | 0.99                | 0.99                | 0.98                | 0.98                | 0.97                | 0.97                | 0.97                | ⋯                   | 0.90                | 0.90                | 0.89                | 0.89                | 0.88                | 0.88                | 0.87                | 0.87                |  8123.17            |  43.62              | \n",
       "| open_office_calc    | 1                   | 1.00                | 0.99                | 0.99                | 0.98                | 0.98                | 0.98                | 0.97                | 0.97                | ⋯                   | 0.91                | 0.90                | 0.90                | 0.89                | 0.89                | 0.88                | 0.88                | 0.87                |  3245.73            |  18.99              | \n",
       "| open_office_draw    | 1                   | 1.00                | 0.99                | 0.98                | 0.98                | 0.97                | 0.97                | 0.96                | 0.96                | ⋯                   | 0.88                | 0.88                | 0.87                | 0.87                | 0.86                | 0.86                | 0.85                | 0.84                |   810.53            |   4.97              | \n",
       "| apache_ant          | 1                   | 1.00                | 0.99                | 0.99                | 0.99                | 0.98                | 0.98                | 0.97                | 0.97                | ⋯                   | 0.91                | 0.91                | 0.91                | 0.90                | 0.90                | 0.89                | 0.89                | 0.88                |  1213.95            |  10.74              | \n",
       "| apache_apache2      | 1                   | 0.99                | 0.97                | 0.96                | 0.94                | 0.93                | 0.91                | 0.90                | 0.88                | ⋯                   | 0.69                | 0.68                | 0.66                | 0.65                | 0.63                | 0.62                | 0.60                | 0.58                |   976.48            |   8.71              | \n",
       "| libre_office_writer | 1                   | 0.99                | 0.98                | 0.97                | 0.96                | 0.95                | 0.94                | 0.93                | 0.92                | ⋯                   | 0.79                | 0.78                | 0.78                | 0.77                | 0.76                | 0.75                | 0.74                | 0.73                |  3245.32            |  28.38              | \n",
       "| libre_office_calc   | 1                   | 0.99                | 0.98                | 0.97                | 0.96                | 0.95                | 0.94                | 0.94                | 0.93                | ⋯                   | 0.79                | 0.78                | 0.77                | 0.76                | 0.75                | 0.75                | 0.74                | 0.73                |  1816.79            |  17.94              | \n",
       "| libre_office_draw   | 1                   | 0.99                | 0.98                | 0.97                | 0.96                | 0.94                | 0.93                | 0.92                | 0.91                | ⋯                   | 0.75                | 0.74                | 0.73                | 0.72                | 0.71                | 0.70                | 0.69                | 0.68                |   370.36            |   3.96              | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "   Products            Acf_lag_1 Acf_lag_2 Acf_lag_3 Acf_lag_4 Acf_lag_5\n",
       "1  kernel_networking   1         0.99      0.99      0.98      0.97     \n",
       "2  eclipse_platform    1         1.00      0.99      0.99      0.98     \n",
       "3  eclipse_data_tools  1         0.99      0.98      0.97      0.96     \n",
       "4  eclipse_birt        1         1.00      1.00      0.99      0.99     \n",
       "5  eclipse_jdt         1         1.00      0.99      0.99      0.99     \n",
       "6  eclipse_pde         1         0.99      0.98      0.98      0.97     \n",
       "7  mozilla_calendar    1         1.00      0.99      0.98      0.98     \n",
       "8  mozilla_firefox     1         1.00      0.99      0.99      0.99     \n",
       "9  mozilla_core        1         1.00      0.99      0.99      0.99     \n",
       "10 mozilla_thunderbird 1         1.00      1.00      0.99      0.99     \n",
       "11 kernel_IO_Storage   1         0.99      0.99      0.98      0.97     \n",
       "12 kernel_file_system  1         0.99      0.98      0.97      0.96     \n",
       "13 open_office_writer  1         1.00      0.99      0.99      0.98     \n",
       "14 open_office_calc    1         1.00      0.99      0.99      0.98     \n",
       "15 open_office_draw    1         1.00      0.99      0.98      0.98     \n",
       "16 apache_ant          1         1.00      0.99      0.99      0.99     \n",
       "17 apache_apache2      1         0.99      0.97      0.96      0.94     \n",
       "18 libre_office_writer 1         0.99      0.98      0.97      0.96     \n",
       "19 libre_office_calc   1         0.99      0.98      0.97      0.96     \n",
       "20 libre_office_draw   1         0.99      0.98      0.97      0.96     \n",
       "   Acf_lag_6 Acf_lag_7 Acf_lag_8 Acf_lag_9 ⋯ Acf_lag_23 Acf_lag_24 Acf_lag_25\n",
       "1  0.97      0.96      0.95      0.95      ⋯ 0.83       0.82       0.81      \n",
       "2  0.98      0.98      0.97      0.97      ⋯ 0.90       0.90       0.89      \n",
       "3  0.95      0.94      0.92      0.91      ⋯ 0.77       0.76       0.75      \n",
       "4  0.98      0.98      0.98      0.97      ⋯ 0.90       0.89       0.89      \n",
       "5  0.98      0.98      0.98      0.97      ⋯ 0.92       0.92       0.91      \n",
       "6  0.97      0.96      0.96      0.95      ⋯ 0.89       0.89       0.88      \n",
       "7  0.97      0.96      0.95      0.95      ⋯ 0.85       0.84       0.83      \n",
       "8  0.99      0.98      0.98      0.98      ⋯ 0.93       0.93       0.93      \n",
       "9  0.98      0.98      0.97      0.97      ⋯ 0.92       0.91       0.91      \n",
       "10 0.99      0.98      0.98      0.98      ⋯ 0.94       0.93       0.93      \n",
       "11 0.97      0.96      0.95      0.94      ⋯ 0.82       0.81       0.80      \n",
       "12 0.95      0.93      0.92      0.91      ⋯ 0.74       0.73       0.71      \n",
       "13 0.98      0.97      0.97      0.97      ⋯ 0.90       0.90       0.89      \n",
       "14 0.98      0.98      0.97      0.97      ⋯ 0.91       0.90       0.90      \n",
       "15 0.97      0.97      0.96      0.96      ⋯ 0.88       0.88       0.87      \n",
       "16 0.98      0.98      0.97      0.97      ⋯ 0.91       0.91       0.91      \n",
       "17 0.93      0.91      0.90      0.88      ⋯ 0.69       0.68       0.66      \n",
       "18 0.95      0.94      0.93      0.92      ⋯ 0.79       0.78       0.78      \n",
       "19 0.95      0.94      0.94      0.93      ⋯ 0.79       0.78       0.77      \n",
       "20 0.94      0.93      0.92      0.91      ⋯ 0.75       0.74       0.73      \n",
       "   Acf_lag_26 Acf_lag_27 Acf_lag_28 Acf_lag_29 Acf_lag_30 Mean_backlog_size\n",
       "1  0.80       0.79       0.78       0.78       0.77         112.38         \n",
       "2  0.89       0.88       0.88       0.87       0.87        7240.95         \n",
       "3  0.74       0.73       0.72       0.71       0.70         159.05         \n",
       "4  0.88       0.88       0.87       0.87       0.86        1000.21         \n",
       "5  0.91       0.90       0.90       0.90       0.89        3329.15         \n",
       "6  0.88       0.87       0.87       0.86       0.86         850.86         \n",
       "7  0.82       0.82       0.81       0.80       0.80        1219.78         \n",
       "8  0.92       0.92       0.92       0.91       0.91       11354.33         \n",
       "9  0.91       0.90       0.90       0.90       0.89       29050.79         \n",
       "10 0.93       0.92       0.92       0.92       0.91        3708.50         \n",
       "11 0.79       0.78       0.77       0.76       0.76         161.42         \n",
       "12 0.70       0.69       0.68       0.67       0.65         190.59         \n",
       "13 0.89       0.88       0.88       0.87       0.87        8123.17         \n",
       "14 0.89       0.89       0.88       0.88       0.87        3245.73         \n",
       "15 0.87       0.86       0.86       0.85       0.84         810.53         \n",
       "16 0.90       0.90       0.89       0.89       0.88        1213.95         \n",
       "17 0.65       0.63       0.62       0.60       0.58         976.48         \n",
       "18 0.77       0.76       0.75       0.74       0.73        3245.32         \n",
       "19 0.76       0.75       0.75       0.74       0.73        1816.79         \n",
       "20 0.72       0.71       0.70       0.69       0.68         370.36         \n",
       "   Mean_backlog_size_change\n",
       "1    2.29                  \n",
       "2   47.60                  \n",
       "3    4.54                  \n",
       "4   18.89                  \n",
       "5   28.73                  \n",
       "6    8.27                  \n",
       "7   12.59                  \n",
       "8   82.95                  \n",
       "9  141.83                  \n",
       "10  22.94                  \n",
       "11   2.59                  \n",
       "12   3.57                  \n",
       "13  43.62                  \n",
       "14  18.99                  \n",
       "15   4.97                  \n",
       "16  10.74                  \n",
       "17   8.71                  \n",
       "18  28.38                  \n",
       "19  17.94                  \n",
       "20   3.96                  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "backlog_measures = read.csv(file=paste(\"data/backlogs_metrics.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "backlog_measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final evaluation and statistical tests "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Products</th><th scope=col>arima_th.all</th><th scope=col>ets_th.all</th><th scope=col>ericsson</th><th scope=col>naive</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>kernel_networking        </td><td>  1.95                   </td><td>  1.93                   </td><td>  2.53                   </td><td>  1.83                   </td></tr>\n",
       "\t<tr><td>eclipse_platform         </td><td> 45.43                   </td><td> 44.05                   </td><td> 46.65                   </td><td> 49.11                   </td></tr>\n",
       "\t<tr><td>eclipse_data_tools       </td><td>  8.12                   </td><td>  8.37                   </td><td>  8.52                   </td><td>  8.75                   </td></tr>\n",
       "\t<tr><td>eclipse_birt             </td><td> 23.51                   </td><td> 20.67                   </td><td> 23.61                   </td><td> 24.15                   </td></tr>\n",
       "\t<tr><td>eclipse_jdt              </td><td> 24.57                   </td><td> 22.17                   </td><td> 26.01                   </td><td> 23.54                   </td></tr>\n",
       "\t<tr><td>eclipse_pde              </td><td>  8.07                   </td><td>  8.24                   </td><td>  9.67                   </td><td>  8.35                   </td></tr>\n",
       "\t<tr><td>mozilla_calendar         </td><td> 18.97                   </td><td> 13.75                   </td><td> 14.86                   </td><td> 14.85                   </td></tr>\n",
       "\t<tr><td>mozilla_firefox          </td><td> 60.17                   </td><td> 58.61                   </td><td> 71.03                   </td><td> 70.12                   </td></tr>\n",
       "\t<tr><td>mozilla_core             </td><td>117.35                   </td><td>125.02                   </td><td>131.98                   </td><td>144.47                   </td></tr>\n",
       "\t<tr><td>mozilla_thunderbird      </td><td> 18.70                   </td><td> 19.52                   </td><td> 20.94                   </td><td> 22.04                   </td></tr>\n",
       "\t<tr><td>kernel_IO_Storage        </td><td>  2.80                   </td><td>  2.77                   </td><td>  3.19                   </td><td>  2.72                   </td></tr>\n",
       "\t<tr><td>kernel_file_system       </td><td>  2.68                   </td><td>  2.74                   </td><td>  3.50                   </td><td>  2.83                   </td></tr>\n",
       "\t<tr><td>open_office_writer       </td><td> 46.25                   </td><td> 44.70                   </td><td> 54.39                   </td><td> 46.26                   </td></tr>\n",
       "\t<tr><td>open_office_calc         </td><td> 17.21                   </td><td> 11.70                   </td><td> 19.18                   </td><td> 13.84                   </td></tr>\n",
       "\t<tr><td>open_office_draw         </td><td>  6.24                   </td><td>  3.84                   </td><td>  4.96                   </td><td>  2.71                   </td></tr>\n",
       "\t<tr><td>apache_ant               </td><td> 20.12                   </td><td> 20.43                   </td><td> 19.79                   </td><td> 22.69                   </td></tr>\n",
       "\t<tr><td>apache_apache2           </td><td>  6.13                   </td><td>  7.36                   </td><td> 10.49                   </td><td>  7.71                   </td></tr>\n",
       "\t<tr><td>libre_office_writer      </td><td> 20.53                   </td><td> 20.90                   </td><td> 24.19                   </td><td> 28.79                   </td></tr>\n",
       "\t<tr><td>libre_office_calc        </td><td> 11.38                   </td><td> 12.01                   </td><td> 12.69                   </td><td> 17.62                   </td></tr>\n",
       "\t<tr><td>libre_office_draw        </td><td>  3.46                   </td><td>  3.51                   </td><td>  3.43                   </td><td>  3.91                   </td></tr>\n",
       "\t<tr><td>mean_error               </td><td> 23.18                   </td><td> 22.61                   </td><td> 25.58                   </td><td> 25.81                   </td></tr>\n",
       "\t<tr><td>errors_standard_deviation</td><td> 27.34                   </td><td> 28.69                   </td><td> 30.99                   </td><td> 33.21                   </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       " Products & arima\\_th.all & ets\\_th.all & ericsson & naive\\\\\n",
       "\\hline\n",
       "\t kernel\\_networking         &   1.95                      &   1.93                      &   2.53                      &   1.83                     \\\\\n",
       "\t eclipse\\_platform          &  45.43                      &  44.05                      &  46.65                      &  49.11                     \\\\\n",
       "\t eclipse\\_data\\_tools        &   8.12                        &   8.37                        &   8.52                        &   8.75                       \\\\\n",
       "\t eclipse\\_birt              &  23.51                      &  20.67                      &  23.61                      &  24.15                     \\\\\n",
       "\t eclipse\\_jdt               &  24.57                      &  22.17                      &  26.01                      &  23.54                     \\\\\n",
       "\t eclipse\\_pde               &   8.07                      &   8.24                      &   9.67                      &   8.35                     \\\\\n",
       "\t mozilla\\_calendar          &  18.97                      &  13.75                      &  14.86                      &  14.85                     \\\\\n",
       "\t mozilla\\_firefox           &  60.17                      &  58.61                      &  71.03                      &  70.12                     \\\\\n",
       "\t mozilla\\_core              & 117.35                      & 125.02                      & 131.98                      & 144.47                     \\\\\n",
       "\t mozilla\\_thunderbird       &  18.70                      &  19.52                      &  20.94                      &  22.04                     \\\\\n",
       "\t kernel\\_IO\\_Storage         &   2.80                        &   2.77                        &   3.19                        &   2.72                       \\\\\n",
       "\t kernel\\_file\\_system        &   2.68                        &   2.74                        &   3.50                        &   2.83                       \\\\\n",
       "\t open\\_office\\_writer        &  46.25                        &  44.70                        &  54.39                        &  46.26                       \\\\\n",
       "\t open\\_office\\_calc          &  17.21                        &  11.70                        &  19.18                        &  13.84                       \\\\\n",
       "\t open\\_office\\_draw          &   6.24                        &   3.84                        &   4.96                        &   2.71                       \\\\\n",
       "\t apache\\_ant                &  20.12                      &  20.43                      &  19.79                      &  22.69                     \\\\\n",
       "\t apache\\_apache2            &   6.13                      &   7.36                      &  10.49                      &   7.71                     \\\\\n",
       "\t libre\\_office\\_writer       &  20.53                        &  20.90                        &  24.19                        &  28.79                       \\\\\n",
       "\t libre\\_office\\_calc         &  11.38                        &  12.01                        &  12.69                        &  17.62                       \\\\\n",
       "\t libre\\_office\\_draw         &   3.46                        &   3.51                        &   3.43                        &   3.91                       \\\\\n",
       "\t mean\\_error                &  23.18                      &  22.61                      &  25.58                      &  25.81                     \\\\\n",
       "\t errors\\_standard\\_deviation &  27.34                        &  28.69                        &  30.99                        &  33.21                       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Products | arima_th.all | ets_th.all | ericsson | naive | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking         |   1.95                    |   1.93                    |   2.53                    |   1.83                    | \n",
       "| eclipse_platform          |  45.43                    |  44.05                    |  46.65                    |  49.11                    | \n",
       "| eclipse_data_tools        |   8.12                    |   8.37                    |   8.52                    |   8.75                    | \n",
       "| eclipse_birt              |  23.51                    |  20.67                    |  23.61                    |  24.15                    | \n",
       "| eclipse_jdt               |  24.57                    |  22.17                    |  26.01                    |  23.54                    | \n",
       "| eclipse_pde               |   8.07                    |   8.24                    |   9.67                    |   8.35                    | \n",
       "| mozilla_calendar          |  18.97                    |  13.75                    |  14.86                    |  14.85                    | \n",
       "| mozilla_firefox           |  60.17                    |  58.61                    |  71.03                    |  70.12                    | \n",
       "| mozilla_core              | 117.35                    | 125.02                    | 131.98                    | 144.47                    | \n",
       "| mozilla_thunderbird       |  18.70                    |  19.52                    |  20.94                    |  22.04                    | \n",
       "| kernel_IO_Storage         |   2.80                    |   2.77                    |   3.19                    |   2.72                    | \n",
       "| kernel_file_system        |   2.68                    |   2.74                    |   3.50                    |   2.83                    | \n",
       "| open_office_writer        |  46.25                    |  44.70                    |  54.39                    |  46.26                    | \n",
       "| open_office_calc          |  17.21                    |  11.70                    |  19.18                    |  13.84                    | \n",
       "| open_office_draw          |   6.24                    |   3.84                    |   4.96                    |   2.71                    | \n",
       "| apache_ant                |  20.12                    |  20.43                    |  19.79                    |  22.69                    | \n",
       "| apache_apache2            |   6.13                    |   7.36                    |  10.49                    |   7.71                    | \n",
       "| libre_office_writer       |  20.53                    |  20.90                    |  24.19                    |  28.79                    | \n",
       "| libre_office_calc         |  11.38                    |  12.01                    |  12.69                    |  17.62                    | \n",
       "| libre_office_draw         |   3.46                    |   3.51                    |   3.43                    |   3.91                    | \n",
       "| mean_error                |  23.18                    |  22.61                    |  25.58                    |  25.81                    | \n",
       "| errors_standard_deviation |  27.34                    |  28.69                    |  30.99                    |  33.21                    | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "   Products                  arima_th.all ets_th.all ericsson naive \n",
       "1  kernel_networking           1.95         1.93       2.53     1.83\n",
       "2  eclipse_platform           45.43        44.05      46.65    49.11\n",
       "3  eclipse_data_tools          8.12         8.37       8.52     8.75\n",
       "4  eclipse_birt               23.51        20.67      23.61    24.15\n",
       "5  eclipse_jdt                24.57        22.17      26.01    23.54\n",
       "6  eclipse_pde                 8.07         8.24       9.67     8.35\n",
       "7  mozilla_calendar           18.97        13.75      14.86    14.85\n",
       "8  mozilla_firefox            60.17        58.61      71.03    70.12\n",
       "9  mozilla_core              117.35       125.02     131.98   144.47\n",
       "10 mozilla_thunderbird        18.70        19.52      20.94    22.04\n",
       "11 kernel_IO_Storage           2.80         2.77       3.19     2.72\n",
       "12 kernel_file_system          2.68         2.74       3.50     2.83\n",
       "13 open_office_writer         46.25        44.70      54.39    46.26\n",
       "14 open_office_calc           17.21        11.70      19.18    13.84\n",
       "15 open_office_draw            6.24         3.84       4.96     2.71\n",
       "16 apache_ant                 20.12        20.43      19.79    22.69\n",
       "17 apache_apache2              6.13         7.36      10.49     7.71\n",
       "18 libre_office_writer        20.53        20.90      24.19    28.79\n",
       "19 libre_office_calc          11.38        12.01      12.69    17.62\n",
       "20 libre_office_draw           3.46         3.51       3.43     3.91\n",
       "21 mean_error                 23.18        22.61      25.58    25.81\n",
       "22 errors_standard_deviation  27.34        28.69      30.99    33.21"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "errors_evaluation = read.csv(file=paste(\"data/errors_evaluation_week_1.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "\n",
    "backlog_measures = read.csv(file=paste(\"data/backlogs_metrics.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "\n",
    "selected_models = errors_evaluation %>% select (arima_th.all, ets_th.all, ericsson, naive)\n",
    "#selected_models <- head(selected_models, -2)\n",
    "\n",
    "backlog_measures = backlog_measures %>% select(Products, Mean_backlog_size, Mean_backlog_size_change)\n",
    "\n",
    "library(data.table)\n",
    "setDT(selected_models, keep.rownames = \"Products\")[]\n",
    "\n",
    "merged_data = merge(selected_models, backlog_measures, by=\"Products\")\n",
    "\n",
    "both_models_worst_than_naive = merged_data %>% filter (arima_th.all>=naive & ets_th.all>=naive) %>% select (Products)                                     \n",
    "only_arima_worst = merged_data %>% filter (arima_th.all>=naive & ets_th.all<=naive) %>% select (Products)\n",
    "models_better_than_naiv = merged_data %>% filter (arima_th.all<naive & ets_th.all<naive) %>% select (Products)\n",
    "\n",
    "arima_errors = selected_models[,\"arima_th.all\"][[1]]\n",
    "ets_errors = selected_models[,\"ets_th.all\"][[1]]\n",
    "ericsson_errors = selected_models[,\"ericsson\"][[1]]\n",
    "naive_errors = selected_models[,\"naive\"][[1]]\n",
    "\n",
    "arima_cd = cliff.delta(naive_errors, arima_errors)[[1]]\n",
    "ets_cd = cliff.delta(naive_errors, ets_errors)[[1]]\n",
    "ericsson_cd = cliff.delta(naive_errors, ericsson_errors)[[1]]\n",
    "nnt = data.frame(Products=\"NNT\",\n",
    "                 arima_th.all = arima_cd^-1,\n",
    "                 ets_th.all = ets_cd^-1, \n",
    "                 ericsson = ericsson_cd^-1,\n",
    "                 naive = 0)\n",
    "f = rbind(selected_models, nnt)\n",
    "\n",
    "write.table(backlog_measures, file =\"data/dupa\", sep=\",\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"arima > naive\"\n",
      "[1] FALSE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > naive\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ericsson > naive\"\n",
      "[1] FALSE\n",
      "[1] \"----------------\"\n",
      "[1] \"arima > ericsson\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > ericsson\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > arima\"\n",
      "[1] FALSE\n",
      "[1] \"----------------\"\n"
     ]
    }
   ],
   "source": [
    "print(\"arima > naive\")\n",
    "p = wilcox.test(selected_models$naive, selected_models$arima_th.all, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > naive\")\n",
    "p = wilcox.test(selected_models$naive, selected_models$ets_th.all, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ericsson > naive\")\n",
    "p = wilcox.test(selected_models$naive, selected_models$ericsson, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"arima > ericsson\")\n",
    "p = wilcox.test(selected_models$arima_th.all, selected_models$ericsson, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > ericsson\")\n",
    "p = wilcox.test(selected_models$ets_th.all, selected_models$ericsson, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > arima\")\n",
    "p = wilcox.test(selected_models$ets_th.all, selected_models$arima_th.all, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Divided backlog predictions summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>products</th><th scope=col>arima_th.all</th><th scope=col>ets_th.all</th><th scope=col>ericsson</th><th scope=col>naive</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>kernel_networking  </td><td> 1.97              </td><td> 2.02              </td><td> 2.30              </td><td>  1.81             </td></tr>\n",
       "\t<tr><td>eclipse_platform   </td><td>44.99              </td><td>42.50              </td><td>43.39              </td><td> 47.60             </td></tr>\n",
       "\t<tr><td>eclipse_data_tools </td><td> 3.43              </td><td> 3.63              </td><td> 3.23              </td><td>  3.63             </td></tr>\n",
       "\t<tr><td>eclipse_birt       </td><td>15.17              </td><td>14.82              </td><td>17.74              </td><td> 16.36             </td></tr>\n",
       "\t<tr><td>eclipse_jdt        </td><td>21.07              </td><td>20.36              </td><td>25.19              </td><td> 22.62             </td></tr>\n",
       "\t<tr><td>eclipse_pde        </td><td> 7.61              </td><td> 7.60              </td><td> 8.63              </td><td>  7.88             </td></tr>\n",
       "\t<tr><td>mozilla_calendar   </td><td> 7.92              </td><td> 7.46              </td><td> 7.78              </td><td>  9.85             </td></tr>\n",
       "\t<tr><td>mozilla_firefox    </td><td>56.81              </td><td>56.45              </td><td>57.44              </td><td> 70.11             </td></tr>\n",
       "\t<tr><td>mozilla_core       </td><td>87.04              </td><td>84.59              </td><td>85.89              </td><td>110.63             </td></tr>\n",
       "\t<tr><td>mozilla_thunderbird</td><td>17.40              </td><td>18.00              </td><td>17.62              </td><td> 21.05             </td></tr>\n",
       "\t<tr><td>kernel_IO_Storage  </td><td> 2.61              </td><td> 2.58              </td><td> 2.88              </td><td>  2.55             </td></tr>\n",
       "\t<tr><td>kernel_file_system </td><td> 2.95              </td><td> 2.88              </td><td> 2.86              </td><td>  2.79             </td></tr>\n",
       "\t<tr><td>open_office_writer </td><td>11.73              </td><td>10.54              </td><td>14.65              </td><td> 23.60             </td></tr>\n",
       "\t<tr><td>open_office_calc   </td><td> 5.74              </td><td> 5.67              </td><td> 7.77              </td><td> 10.69             </td></tr>\n",
       "\t<tr><td>open_office_draw   </td><td> 2.23              </td><td> 2.16              </td><td> 2.89              </td><td>  2.71             </td></tr>\n",
       "\t<tr><td>apache_ant         </td><td> 3.35              </td><td> 3.01              </td><td> 3.42              </td><td>  6.27             </td></tr>\n",
       "\t<tr><td>apache_apache2     </td><td> 4.53              </td><td> 4.59              </td><td> 5.82              </td><td>  6.23             </td></tr>\n",
       "\t<tr><td>libre_office_writer</td><td>20.88              </td><td>21.73              </td><td>20.76              </td><td> 27.97             </td></tr>\n",
       "\t<tr><td>libre_office_calc  </td><td>13.31              </td><td>12.94              </td><td>12.10              </td><td> 17.57             </td></tr>\n",
       "\t<tr><td>libre_office_draw  </td><td> 3.24              </td><td> 3.63              </td><td> 3.32              </td><td>  3.81             </td></tr>\n",
       "\t<tr><td>mean               </td><td>16.70              </td><td>16.36              </td><td>17.28              </td><td> 20.79             </td></tr>\n",
       "\t<tr><td>standard deviation </td><td>22.02              </td><td>21.44              </td><td>21.68              </td><td> 27.20             </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllll}\n",
       " products & arima\\_th.all & ets\\_th.all & ericsson & naive\\\\\n",
       "\\hline\n",
       "\t kernel\\_networking   &  1.97                 &  2.02                 &  2.30                 &   1.81               \\\\\n",
       "\t eclipse\\_platform    & 44.99                 & 42.50                 & 43.39                 &  47.60               \\\\\n",
       "\t eclipse\\_data\\_tools  &  3.43                   &  3.63                   &  3.23                   &   3.63                 \\\\\n",
       "\t eclipse\\_birt        & 15.17                 & 14.82                 & 17.74                 &  16.36               \\\\\n",
       "\t eclipse\\_jdt         & 21.07                 & 20.36                 & 25.19                 &  22.62               \\\\\n",
       "\t eclipse\\_pde         &  7.61                 &  7.60                 &  8.63                 &   7.88               \\\\\n",
       "\t mozilla\\_calendar    &  7.92                 &  7.46                 &  7.78                 &   9.85               \\\\\n",
       "\t mozilla\\_firefox     & 56.81                 & 56.45                 & 57.44                 &  70.11               \\\\\n",
       "\t mozilla\\_core        & 87.04                 & 84.59                 & 85.89                 & 110.63               \\\\\n",
       "\t mozilla\\_thunderbird & 17.40                 & 18.00                 & 17.62                 &  21.05               \\\\\n",
       "\t kernel\\_IO\\_Storage   &  2.61                   &  2.58                   &  2.88                   &   2.55                 \\\\\n",
       "\t kernel\\_file\\_system  &  2.95                   &  2.88                   &  2.86                   &   2.79                 \\\\\n",
       "\t open\\_office\\_writer  & 11.73                   & 10.54                   & 14.65                   &  23.60                 \\\\\n",
       "\t open\\_office\\_calc    &  5.74                   &  5.67                   &  7.77                   &  10.69                 \\\\\n",
       "\t open\\_office\\_draw    &  2.23                   &  2.16                   &  2.89                   &   2.71                 \\\\\n",
       "\t apache\\_ant          &  3.35                 &  3.01                 &  3.42                 &   6.27               \\\\\n",
       "\t apache\\_apache2      &  4.53                 &  4.59                 &  5.82                 &   6.23               \\\\\n",
       "\t libre\\_office\\_writer & 20.88                   & 21.73                   & 20.76                   &  27.97                 \\\\\n",
       "\t libre\\_office\\_calc   & 13.31                   & 12.94                   & 12.10                   &  17.57                 \\\\\n",
       "\t libre\\_office\\_draw   &  3.24                   &  3.63                   &  3.32                   &   3.81                 \\\\\n",
       "\t mean                & 16.70               & 16.36               & 17.28               &  20.79             \\\\\n",
       "\t standard deviation  & 22.02               & 21.44               & 21.68               &  27.20             \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "products | arima_th.all | ets_th.all | ericsson | naive | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking   |  1.97               |  2.02               |  2.30               |   1.81              | \n",
       "| eclipse_platform    | 44.99               | 42.50               | 43.39               |  47.60              | \n",
       "| eclipse_data_tools  |  3.43               |  3.63               |  3.23               |   3.63              | \n",
       "| eclipse_birt        | 15.17               | 14.82               | 17.74               |  16.36              | \n",
       "| eclipse_jdt         | 21.07               | 20.36               | 25.19               |  22.62              | \n",
       "| eclipse_pde         |  7.61               |  7.60               |  8.63               |   7.88              | \n",
       "| mozilla_calendar    |  7.92               |  7.46               |  7.78               |   9.85              | \n",
       "| mozilla_firefox     | 56.81               | 56.45               | 57.44               |  70.11              | \n",
       "| mozilla_core        | 87.04               | 84.59               | 85.89               | 110.63              | \n",
       "| mozilla_thunderbird | 17.40               | 18.00               | 17.62               |  21.05              | \n",
       "| kernel_IO_Storage   |  2.61               |  2.58               |  2.88               |   2.55              | \n",
       "| kernel_file_system  |  2.95               |  2.88               |  2.86               |   2.79              | \n",
       "| open_office_writer  | 11.73               | 10.54               | 14.65               |  23.60              | \n",
       "| open_office_calc    |  5.74               |  5.67               |  7.77               |  10.69              | \n",
       "| open_office_draw    |  2.23               |  2.16               |  2.89               |   2.71              | \n",
       "| apache_ant          |  3.35               |  3.01               |  3.42               |   6.27              | \n",
       "| apache_apache2      |  4.53               |  4.59               |  5.82               |   6.23              | \n",
       "| libre_office_writer | 20.88               | 21.73               | 20.76               |  27.97              | \n",
       "| libre_office_calc   | 13.31               | 12.94               | 12.10               |  17.57              | \n",
       "| libre_office_draw   |  3.24               |  3.63               |  3.32               |   3.81              | \n",
       "| mean                | 16.70               | 16.36               | 17.28               |  20.79              | \n",
       "| standard deviation  | 22.02               | 21.44               | 21.68               |  27.20              | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "   products            arima_th.all ets_th.all ericsson naive \n",
       "1  kernel_networking    1.97         2.02       2.30      1.81\n",
       "2  eclipse_platform    44.99        42.50      43.39     47.60\n",
       "3  eclipse_data_tools   3.43         3.63       3.23      3.63\n",
       "4  eclipse_birt        15.17        14.82      17.74     16.36\n",
       "5  eclipse_jdt         21.07        20.36      25.19     22.62\n",
       "6  eclipse_pde          7.61         7.60       8.63      7.88\n",
       "7  mozilla_calendar     7.92         7.46       7.78      9.85\n",
       "8  mozilla_firefox     56.81        56.45      57.44     70.11\n",
       "9  mozilla_core        87.04        84.59      85.89    110.63\n",
       "10 mozilla_thunderbird 17.40        18.00      17.62     21.05\n",
       "11 kernel_IO_Storage    2.61         2.58       2.88      2.55\n",
       "12 kernel_file_system   2.95         2.88       2.86      2.79\n",
       "13 open_office_writer  11.73        10.54      14.65     23.60\n",
       "14 open_office_calc     5.74         5.67       7.77     10.69\n",
       "15 open_office_draw     2.23         2.16       2.89      2.71\n",
       "16 apache_ant           3.35         3.01       3.42      6.27\n",
       "17 apache_apache2       4.53         4.59       5.82      6.23\n",
       "18 libre_office_writer 20.88        21.73      20.76     27.97\n",
       "19 libre_office_calc   13.31        12.94      12.10     17.57\n",
       "20 libre_office_draw    3.24         3.63       3.32      3.81\n",
       "21 mean                16.70        16.36      17.28     20.79\n",
       "22 standard deviation  22.02        21.44      21.68     27.20"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>products</th><th scope=col>arima_th.all</th><th scope=col>ets_th.all</th><th scope=col>ericsson</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>kernel_networking  </td><td>-8.84              </td><td>-11.60             </td><td>-27.07             </td></tr>\n",
       "\t<tr><td>eclipse_platform   </td><td> 5.48              </td><td> 10.71             </td><td>  8.84             </td></tr>\n",
       "\t<tr><td>eclipse_data_tools </td><td> 5.51              </td><td>  0.00             </td><td> 11.02             </td></tr>\n",
       "\t<tr><td>eclipse_birt       </td><td> 7.27              </td><td>  9.41             </td><td> -8.44             </td></tr>\n",
       "\t<tr><td>eclipse_jdt        </td><td> 6.85              </td><td>  9.99             </td><td>-11.36             </td></tr>\n",
       "\t<tr><td>eclipse_pde        </td><td> 3.43              </td><td>  3.55             </td><td> -9.52             </td></tr>\n",
       "\t<tr><td>mozilla_calendar   </td><td>19.59              </td><td> 24.26             </td><td> 21.02             </td></tr>\n",
       "\t<tr><td>mozilla_firefox    </td><td>18.97              </td><td> 19.48             </td><td> 18.07             </td></tr>\n",
       "\t<tr><td>mozilla_core       </td><td>21.32              </td><td> 23.54             </td><td> 22.36             </td></tr>\n",
       "\t<tr><td>mozilla_thunderbird</td><td>17.34              </td><td> 14.49             </td><td> 16.29             </td></tr>\n",
       "\t<tr><td>kernel_IO_Storage  </td><td>-2.35              </td><td> -1.18             </td><td>-12.94             </td></tr>\n",
       "\t<tr><td>kernel_file_system </td><td>-5.73              </td><td> -3.23             </td><td> -2.51             </td></tr>\n",
       "\t<tr><td>open_office_writer </td><td>50.30              </td><td> 55.34             </td><td> 37.92             </td></tr>\n",
       "\t<tr><td>open_office_calc   </td><td>46.30              </td><td> 46.96             </td><td> 27.32             </td></tr>\n",
       "\t<tr><td>open_office_draw   </td><td>17.71              </td><td> 20.30             </td><td> -6.64             </td></tr>\n",
       "\t<tr><td>apache_ant         </td><td>46.57              </td><td> 51.99             </td><td> 45.45             </td></tr>\n",
       "\t<tr><td>apache_apache2     </td><td>27.29              </td><td> 26.32             </td><td>  6.58             </td></tr>\n",
       "\t<tr><td>libre_office_writer</td><td>25.35              </td><td> 22.31             </td><td> 25.78             </td></tr>\n",
       "\t<tr><td>libre_office_calc  </td><td>24.25              </td><td> 26.35             </td><td> 31.13             </td></tr>\n",
       "\t<tr><td>libre_office_draw  </td><td>14.96              </td><td>  4.72             </td><td> 12.86             </td></tr>\n",
       "\t<tr><td>mean               </td><td>17.08              </td><td> 17.69             </td><td> 10.31             </td></tr>\n",
       "\t<tr><td>standard deviation </td><td>16.72              </td><td> 18.11             </td><td> 19.11             </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llll}\n",
       " products & arima\\_th.all & ets\\_th.all & ericsson\\\\\n",
       "\\hline\n",
       "\t kernel\\_networking   & -8.84                 & -11.60                & -27.07               \\\\\n",
       "\t eclipse\\_platform    &  5.48                 &  10.71                &   8.84               \\\\\n",
       "\t eclipse\\_data\\_tools  &  5.51                   &   0.00                  &  11.02                 \\\\\n",
       "\t eclipse\\_birt        &  7.27                 &   9.41                &  -8.44               \\\\\n",
       "\t eclipse\\_jdt         &  6.85                 &   9.99                & -11.36               \\\\\n",
       "\t eclipse\\_pde         &  3.43                 &   3.55                &  -9.52               \\\\\n",
       "\t mozilla\\_calendar    & 19.59                 &  24.26                &  21.02               \\\\\n",
       "\t mozilla\\_firefox     & 18.97                 &  19.48                &  18.07               \\\\\n",
       "\t mozilla\\_core        & 21.32                 &  23.54                &  22.36               \\\\\n",
       "\t mozilla\\_thunderbird & 17.34                 &  14.49                &  16.29               \\\\\n",
       "\t kernel\\_IO\\_Storage   & -2.35                   &  -1.18                  & -12.94                 \\\\\n",
       "\t kernel\\_file\\_system  & -5.73                   &  -3.23                  &  -2.51                 \\\\\n",
       "\t open\\_office\\_writer  & 50.30                   &  55.34                  &  37.92                 \\\\\n",
       "\t open\\_office\\_calc    & 46.30                   &  46.96                  &  27.32                 \\\\\n",
       "\t open\\_office\\_draw    & 17.71                   &  20.30                  &  -6.64                 \\\\\n",
       "\t apache\\_ant          & 46.57                 &  51.99                &  45.45               \\\\\n",
       "\t apache\\_apache2      & 27.29                 &  26.32                &   6.58               \\\\\n",
       "\t libre\\_office\\_writer & 25.35                   &  22.31                  &  25.78                 \\\\\n",
       "\t libre\\_office\\_calc   & 24.25                   &  26.35                  &  31.13                 \\\\\n",
       "\t libre\\_office\\_draw   & 14.96                   &   4.72                  &  12.86                 \\\\\n",
       "\t mean                & 17.08               &  17.69              &  10.31             \\\\\n",
       "\t standard deviation  & 16.72               &  18.11              &  19.11             \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "products | arima_th.all | ets_th.all | ericsson | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking   | -8.84               | -11.60              | -27.07              | \n",
       "| eclipse_platform    |  5.48               |  10.71              |   8.84              | \n",
       "| eclipse_data_tools  |  5.51               |   0.00              |  11.02              | \n",
       "| eclipse_birt        |  7.27               |   9.41              |  -8.44              | \n",
       "| eclipse_jdt         |  6.85               |   9.99              | -11.36              | \n",
       "| eclipse_pde         |  3.43               |   3.55              |  -9.52              | \n",
       "| mozilla_calendar    | 19.59               |  24.26              |  21.02              | \n",
       "| mozilla_firefox     | 18.97               |  19.48              |  18.07              | \n",
       "| mozilla_core        | 21.32               |  23.54              |  22.36              | \n",
       "| mozilla_thunderbird | 17.34               |  14.49              |  16.29              | \n",
       "| kernel_IO_Storage   | -2.35               |  -1.18              | -12.94              | \n",
       "| kernel_file_system  | -5.73               |  -3.23              |  -2.51              | \n",
       "| open_office_writer  | 50.30               |  55.34              |  37.92              | \n",
       "| open_office_calc    | 46.30               |  46.96              |  27.32              | \n",
       "| open_office_draw    | 17.71               |  20.30              |  -6.64              | \n",
       "| apache_ant          | 46.57               |  51.99              |  45.45              | \n",
       "| apache_apache2      | 27.29               |  26.32              |   6.58              | \n",
       "| libre_office_writer | 25.35               |  22.31              |  25.78              | \n",
       "| libre_office_calc   | 24.25               |  26.35              |  31.13              | \n",
       "| libre_office_draw   | 14.96               |   4.72              |  12.86              | \n",
       "| mean                | 17.08               |  17.69              |  10.31              | \n",
       "| standard deviation  | 16.72               |  18.11              |  19.11              | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "   products            arima_th.all ets_th.all ericsson\n",
       "1  kernel_networking   -8.84        -11.60     -27.07  \n",
       "2  eclipse_platform     5.48         10.71       8.84  \n",
       "3  eclipse_data_tools   5.51          0.00      11.02  \n",
       "4  eclipse_birt         7.27          9.41      -8.44  \n",
       "5  eclipse_jdt          6.85          9.99     -11.36  \n",
       "6  eclipse_pde          3.43          3.55      -9.52  \n",
       "7  mozilla_calendar    19.59         24.26      21.02  \n",
       "8  mozilla_firefox     18.97         19.48      18.07  \n",
       "9  mozilla_core        21.32         23.54      22.36  \n",
       "10 mozilla_thunderbird 17.34         14.49      16.29  \n",
       "11 kernel_IO_Storage   -2.35         -1.18     -12.94  \n",
       "12 kernel_file_system  -5.73         -3.23      -2.51  \n",
       "13 open_office_writer  50.30         55.34      37.92  \n",
       "14 open_office_calc    46.30         46.96      27.32  \n",
       "15 open_office_draw    17.71         20.30      -6.64  \n",
       "16 apache_ant          46.57         51.99      45.45  \n",
       "17 apache_apache2      27.29         26.32       6.58  \n",
       "18 libre_office_writer 25.35         22.31      25.78  \n",
       "19 libre_office_calc   24.25         26.35      31.13  \n",
       "20 libre_office_draw   14.96          4.72      12.86  \n",
       "21 mean                17.08         17.69      10.31  \n",
       "22 standard deviation  16.72         18.11      19.11  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product_names = c()\n",
    "arima_errors = c()\n",
    "arima_improvements = c()\n",
    "ets_errors = c()\n",
    "ets_improvements = c()\n",
    "ets_impr = c()\n",
    "ericsson_errors = c()\n",
    "ericsson_improvements = c()\n",
    "\n",
    "naive_errors = c()\n",
    "for(i in 1:nrow(products)){\n",
    "    product = products[i,]\n",
    "    product_names = c(product_names, paste(product[1], product[2], sep=\"_\"))\n",
    "    divided_predictions_evaluation = read.csv(file=paste(\"data/\", product[1], \"/\", product[2], \"/Predictions/Divided_backlog_predictions/errors_evaluation.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "    \n",
    "    ericsson_error = (divided_predictions_evaluation %>% filter (Method==\"ericsson\") %>% select (Error_1))[[1]]\n",
    "    ericsson_improvement = (divided_predictions_evaluation %>% filter (Method==\"ericsson\") %>% select (Impr_1))[[1]]\n",
    "    ericsson_errors = c(ericsson_errors, ericsson_error)\n",
    "    ericsson_improvements = c(ericsson_improvements, ericsson_improvement)\n",
    "    \n",
    "    arima_error = (divided_predictions_evaluation %>% filter (Method==\"arima_th:all\") %>% select (Error_1))[[1]]\n",
    "    arima_improvement = (divided_predictions_evaluation %>% filter (Method==\"arima_th:all\") %>% select (Impr_1))[[1]]\n",
    "    arima_errors = c(arima_errors, arima_error)\n",
    "    arima_improvements = c(arima_improvements, arima_improvement)\n",
    "    \n",
    "    ets_error = (divided_predictions_evaluation %>% filter (Method==\"ets_th:all\") %>% select (Error_1))[[1]]\n",
    "    ets_improvement = (divided_predictions_evaluation %>% filter (Method==\"ets_th:all\") %>% select (Impr_1))[[1]]\n",
    "    ets_errors = c(ets_errors, ets_error)\n",
    "    ets_improvements = c(ets_improvements, ets_improvement)\n",
    "    \n",
    "    naive_error = (divided_predictions_evaluation %>% filter (Method==\"naive\") %>% select (Error_1))[[1]]\n",
    "    naive_errors = c(naive_errors, naive_error)\n",
    "}\n",
    "product_names = c(product_names, \"mean\")\n",
    "product_names = c(product_names, \"standard deviation\")\n",
    "naive_errors = c(naive_errors, round(mean(naive_errors), 2))\n",
    "ets_errors = c(ets_errors, round(mean(ets_errors), 2))\n",
    "arima_errors = c(arima_errors, round(mean(arima_errors), 2))      \n",
    "ericsson_errors = c(ericsson_errors, round(mean(ericsson_errors), 2))\n",
    "ericsson_improvements = c(ericsson_improvements, round(mean(ericsson_improvements), 2))\n",
    "arima_improvements = c(arima_improvements, round(mean(arima_improvements), 2))\n",
    "ets_improvements = c(ets_improvements, round(mean(ets_improvements), 2))   \n",
    "\n",
    "naive_errors = c(naive_errors, round(sd(head(naive_errors, -1)), 2))\n",
    "ets_errors = c(ets_errors, round(sd(head(ets_errors, -1)), 2))\n",
    "arima_errors = c(arima_errors, round(sd(head(arima_errors, -1)), 2))      \n",
    "ericsson_errors = c(ericsson_errors, round(sd(head(ericsson_errors, -1)), 2))\n",
    "ericsson_improvements = c(ericsson_improvements, round(sd(head(ericsson_improvements, -1)), 2))\n",
    "arima_improvements = c(arima_improvements, round(sd(head(arima_improvements, -1)), 2))\n",
    "ets_improvements = c(ets_improvements, round(sd(head(ets_improvements, -1)), 2)) \n",
    "                 \n",
    "                 \n",
    "errors_after_division = data.frame(products = product_names,\n",
    "                           arima_th.all = arima_errors,\n",
    "                           ets_th.all = ets_errors,\n",
    "                           ericsson = ericsson_errors,\n",
    "                           naive = naive_errors)\n",
    "\n",
    "improvements_after_division = data.frame(products = product_names,\n",
    "                       arima_th.all = arima_improvements,\n",
    "                       ets_th.all = ets_improvements,\n",
    "                       ericsson = ericsson_improvements)\n",
    "\n",
    "errors_after_division\n",
    "improvements_after_division\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tests after division "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"arima > naive\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > naive\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in wilcox.test.default(errors_after_division$naive, errors_after_division$ets_th.all, :\n",
      "“cannot compute exact p-value with zeroes”"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ericsson > naive\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"arima > ericsson\"\n",
      "[1] FALSE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > ericsson\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > arima\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in wilcox.test.default(errors_after_division$ets_th.all, errors_after_division$arima_th.all, :\n",
      "“cannot compute exact p-value with ties”"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] FALSE\n",
      "[1] \"----------------\"\n"
     ]
    }
   ],
   "source": [
    "print(\"arima > naive\")\n",
    "p = wilcox.test(errors_after_division$naive, errors_after_division$arima_th.all, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > naive\")\n",
    "p = wilcox.test(errors_after_division$naive, errors_after_division$ets_th.all, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ericsson > naive\")\n",
    "p = wilcox.test(errors_after_division$naive, errors_after_division$ericsson, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"arima > ericsson\")\n",
    "p = wilcox.test(errors_after_division$arima_th.all, errors_after_division$ericsson, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > ericsson\")\n",
    "p = wilcox.test(errors_after_division$ets_th.all, errors_after_division$ericsson, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > arima\")\n",
    "p = wilcox.test(errors_after_division$ets_th.all, errors_after_division$arima_th.all, paired=TRUE)$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"arima > naive\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > naive\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in wilcox.test.default(errors_after_division$naive, errors_after_division$ets_th.all, :\n",
      "“cannot compute exact p-value with zeroes”"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"ericsson > naive\"\n",
      "[1] TRUE\n",
      "[1] \"----------------\"\n",
      "[1] \"arima > ericsson\"\n",
      "[1] FALSE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > ericsson\"\n",
      "[1] FALSE\n",
      "[1] \"----------------\"\n",
      "[1] \"ets > arima\"\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in wilcox.test.default(errors_after_division$ets_th.all, errors_after_division$arima_th.all, :\n",
      "“cannot compute exact p-value with ties”"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] FALSE\n",
      "[1] \"----------------\"\n"
     ]
    }
   ],
   "source": [
    "print(\"arima > naive\")\n",
    "p = wilcox.test(errors_after_division$naive, errors_after_division$arima_th.all, paired=TRUE, alternative = \"greater\")$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > naive\")\n",
    "p = wilcox.test(errors_after_division$naive, errors_after_division$ets_th.all, paired=TRUE, alternative = \"greater\")$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ericsson > naive\")\n",
    "p = wilcox.test(errors_after_division$naive, errors_after_division$ericsson, paired=TRUE, alternative = \"greater\")$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"arima > ericsson\")\n",
    "p = wilcox.test(errors_after_division$arima_th.all, errors_after_division$ericsson, paired=TRUE, alternative = \"greater\")$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > ericsson\")\n",
    "p = wilcox.test(errors_after_division$ets_th.all, errors_after_division$ericsson, paired=TRUE, alternative = \"greater\")$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")\n",
    "print(\"ets > arima\")\n",
    "p = wilcox.test(errors_after_division$ets_th.all, errors_after_division$arima_th.all, paired=TRUE, alternative = \"greater\")$p.value\n",
    "print(p<0.05)\n",
    "print(\"----------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in wilcox.test.default(a, b, paired = TRUE):\n",
      "“cannot compute exact p-value with ties”"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "\tWilcoxon signed rank test with continuity correction\n",
       "\n",
       "data:  a and b\n",
       "V = 1, p-value = 0.4142\n",
       "alternative hypothesis: true location shift is not equal to 0\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = c(1,2,6)\n",
    "b = c(3,4,5)\n",
    "\n",
    "p = wilcox.test(a, b, paired=TRUE)\n",
    "p\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "summarize_product_predictions <- function(product){\n",
    "    product_name = split_product_name(product)       \n",
    "    predictions_evaluation = read.csv(file=paste(\"data/\", product_name[1], \"/\", product_name[2], \"/predictions_evaluation.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "    predictions_evaluation = predictions_evaluation %>% filter (Method == \"arima_th:all\" | Method == \"ericsson\" | Method == \"naive\" | Method == \"ets_th:all\" )\n",
    "    display(product_name)\n",
    "    display(predictions_evaluation)\n",
    "    \n",
    "    divided_predictions_evaluation = read.csv(file=paste(\"data/\", product_name[1], \"/\", product_name[2], \"/Predictions/Divided_backlog_predictions/errors_evaluation.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "    display(\"Divided_backlog_predictions:\")\n",
    "    display(divided_predictions_evaluation)\n",
    "}\n",
    "\n",
    "split_product_name <- function(product_full_name){\n",
    "    product_splitted_name <- c()\n",
    "    splitted_parts = (strsplit(product_full_name, \"_\", fixed=TRUE))[[1]]\n",
    "    if(startsWith(product_full_name, \"libre\") | startsWith(product_full_name, \"open\")){\n",
    "        name_first_part = paste(splitted_parts[1], splitted_parts[2], sep=\"_\")\n",
    "        name_second_part = splitted_parts[3]\n",
    "    } else {\n",
    "        name_first_part = splitted_parts[1]\n",
    "        if(length(splitted_parts)>2){\n",
    "            name_second_part = paste(splitted_parts[2], splitted_parts[3], sep=\"_\")\n",
    "        } else {\n",
    "            name_second_part = splitted_parts[2]\n",
    "        }\n",
    "    }\n",
    "    return(c(name_first_part, name_second_part))\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Backlogs poorly predicted by ARIMA and ETS:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Products</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>kernel_IO_Storage</td></tr>\n",
       "\t<tr><td>kernel_networking</td></tr>\n",
       "\t<tr><td>open_office_draw </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|l}\n",
       " Products\\\\\n",
       "\\hline\n",
       "\t kernel\\_IO\\_Storage\\\\\n",
       "\t kernel\\_networking\\\\\n",
       "\t open\\_office\\_draw \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Products | \n",
       "|---|---|---|\n",
       "| kernel_IO_Storage | \n",
       "| kernel_networking | \n",
       "| open_office_draw  | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Products         \n",
       "1 kernel_IO_Storage\n",
       "2 kernel_networking\n",
       "3 open_office_draw "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "both_models_worst_than_naive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'kernel'</li>\n",
       "\t<li>'IO_Storage'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'kernel'\n",
       "\\item 'IO\\_Storage'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'kernel'\n",
       "2. 'IO_Storage'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"kernel\"     \"IO_Storage\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>2.80        </td><td> -2.94      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.19        </td><td>-17.28      </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>2.77        </td><td> -1.84      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>2.72        </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.80           &  -2.94         & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t ericsson     & 3.19         & -17.28       & negligible   & negligible   &  TRUE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 2.77           &  -1.84         & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t naive        & 2.72         &   0.00       & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 2.80         |  -2.94       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ericsson     | 3.19         | -17.28       | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| ets_th:all   | 2.77         |  -1.84       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| naive        | 2.72         |   0.00       | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 2.80     -2.94 negligible  negligible FALSE        \n",
       "2 ericsson     3.19    -17.28 negligible  negligible  TRUE        \n",
       "3 ets_th:all   2.77     -1.84 negligible  negligible FALSE        \n",
       "4 naive        2.72      0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>2.61        </td><td> -2.35      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>2.58        </td><td> -1.18      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>2.88        </td><td>-12.94      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>2.55        </td><td>  0.00      </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.61           &  -2.35         & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 2.58           &  -1.18         & negligible     & negligible     & FALSE         \\\\\n",
       "\t ericsson     & 2.88         & -12.94       & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 2.55         &   0.00       & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 2.61         |  -2.35       | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 2.58         |  -1.18       | negligible   | negligible   | FALSE        | \n",
       "| ericsson     | 2.88         | -12.94       | negligible   | negligible   | FALSE        | \n",
       "| naive        | 2.55         |   0.00       | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 2.61     -2.35 negligible  negligible FALSE        \n",
       "2 ets_th:all   2.58     -1.18 negligible  negligible FALSE        \n",
       "3 ericsson     2.88    -12.94 negligible  negligible FALSE        \n",
       "4 naive        2.55      0.00 NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "product = toString(both_models_worst_than_naive$Products[1])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'kernel'</li>\n",
       "\t<li>'networking'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'kernel'\n",
       "\\item 'networking'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'kernel'\n",
       "2. 'networking'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"kernel\"     \"networking\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>1.95        </td><td> -6.56      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>2.53        </td><td>-38.25      </td><td>small       </td><td>small       </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>1.93        </td><td> -5.46      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>1.83        </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 1.95           &  -6.56         & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t ericsson     & 2.53         & -38.25       & small        & small        &  TRUE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 1.93           &  -5.46         & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t naive        & 1.83         &   0.00       & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 1.95         |  -6.56       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ericsson     | 2.53         | -38.25       | small        | small        |  TRUE        | TRUE         | \n",
       "| ets_th:all   | 1.93         |  -5.46       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| naive        | 1.83         |   0.00       | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 1.95     -6.56 negligible  negligible FALSE        \n",
       "2 ericsson     2.53    -38.25 small       small       TRUE        \n",
       "3 ets_th:all   1.93     -5.46 negligible  negligible FALSE        \n",
       "4 naive        1.83      0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>1.97        </td><td> -8.84      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>2.02        </td><td>-11.60      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>2.30        </td><td>-27.07      </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>1.81        </td><td>  0.00      </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 1.97           &  -8.84         & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 2.02           & -11.60         & negligible     & negligible     & FALSE         \\\\\n",
       "\t ericsson     & 2.30         & -27.07       & small        & small        &  TRUE       \\\\\n",
       "\t naive        & 1.81         &   0.00       & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 1.97         |  -8.84       | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 2.02         | -11.60       | negligible   | negligible   | FALSE        | \n",
       "| ericsson     | 2.30         | -27.07       | small        | small        |  TRUE        | \n",
       "| naive        | 1.81         |   0.00       | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 1.97     -8.84 negligible  negligible FALSE        \n",
       "2 ets_th:all   2.02    -11.60 negligible  negligible FALSE        \n",
       "3 ericsson     2.30    -27.07 small       small       TRUE        \n",
       "4 naive        1.81      0.00 NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(both_models_worst_than_naive$Products[2])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'open_office'</li>\n",
       "\t<li>'draw'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'open\\_office'\n",
       "\\item 'draw'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'open_office'\n",
       "2. 'draw'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"open_office\" \"draw\"       "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>6.24        </td><td>-130.26     </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>4.96        </td><td> -83.03     </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>3.84        </td><td> -41.70     </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>2.71        </td><td>   0.00     </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 6.24           & -130.26        & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t ericsson     & 4.96         &  -83.03      & negligible   & negligible   & FALSE        & FALSE       \\\\\n",
       "\t ets\\_th:all   & 3.84           &  -41.70        & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t naive        & 2.71         &    0.00      & negligible   & negligible   &    NA        & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 6.24         | -130.26      | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ericsson     | 4.96         |  -83.03      | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ets_th:all   | 3.84         |  -41.70      | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| naive        | 2.71         |    0.00      | negligible   | negligible   |    NA        | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1  Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 6.24    -130.26 negligible  negligible FALSE        \n",
       "2 ericsson     4.96     -83.03 negligible  negligible FALSE        \n",
       "3 ets_th:all   3.84     -41.70 negligible  negligible FALSE        \n",
       "4 naive        2.71       0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>2.23        </td><td>17.71       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>2.16        </td><td>20.30       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>2.89        </td><td>-6.64       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>2.71        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.23           & 17.71          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 2.16           & 20.30          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ericsson     & 2.89         & -6.64        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 2.71         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 2.23         | 17.71        | negligible   | negligible   |  TRUE        | \n",
       "| ets_th:all   | 2.16         | 20.30        | negligible   | negligible   |  TRUE        | \n",
       "| ericsson     | 2.89         | -6.64        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 2.71         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 2.23    17.71  negligible  negligible  TRUE        \n",
       "2 ets_th:all   2.16    20.30  negligible  negligible  TRUE        \n",
       "3 ericsson     2.89    -6.64  negligible  negligible FALSE        \n",
       "4 naive        2.71     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "product = toString(both_models_worst_than_naive$Products[3])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Backlogs poorly predicted by ARIMA:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Products</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>eclipse_jdt     </td></tr>\n",
       "\t<tr><td>mozilla_calendar</td></tr>\n",
       "\t<tr><td>open_office_calc</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|l}\n",
       " Products\\\\\n",
       "\\hline\n",
       "\t eclipse\\_jdt     \\\\\n",
       "\t mozilla\\_calendar\\\\\n",
       "\t open\\_office\\_calc\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Products | \n",
       "|---|---|---|\n",
       "| eclipse_jdt      | \n",
       "| mozilla_calendar | \n",
       "| open_office_calc | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Products        \n",
       "1 eclipse_jdt     \n",
       "2 mozilla_calendar\n",
       "3 open_office_calc"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "only_arima_worst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'jdt'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'jdt'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'jdt'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\" \"jdt\"    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>24.57       </td><td> -4.38      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>26.01       </td><td>-10.49      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>22.17       </td><td>  5.82      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>23.54       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 24.57          &  -4.38         & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t ericsson     & 26.01        & -10.49       & negligible   & negligible   & FALSE        & FALSE       \\\\\n",
       "\t ets\\_th:all   & 22.17          &   5.82         & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t naive        & 23.54        &   0.00       & negligible   & negligible   &    NA        & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 24.57        |  -4.38       | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ericsson     | 26.01        | -10.49       | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ets_th:all   | 22.17        |   5.82       | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| naive        | 23.54        |   0.00       | negligible   | negligible   |    NA        | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 24.57    -4.38 negligible  negligible FALSE        \n",
       "2 ericsson     26.01   -10.49 negligible  negligible FALSE        \n",
       "3 ets_th:all   22.17     5.82 negligible  negligible FALSE        \n",
       "4 naive        23.54     0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>21.07       </td><td>  6.85      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>20.36       </td><td>  9.99      </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>25.19       </td><td>-11.36      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>22.62       </td><td>  0.00      </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 21.07          &   6.85         & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 20.36          &   9.99         & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ericsson     & 25.19        & -11.36       & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 22.62        &   0.00       & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 21.07        |   6.85       | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 20.36        |   9.99       | negligible   | negligible   |  TRUE        | \n",
       "| ericsson     | 25.19        | -11.36       | negligible   | negligible   | FALSE        | \n",
       "| naive        | 22.62        |   0.00       | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 21.07     6.85 negligible  negligible FALSE        \n",
       "2 ets_th:all   20.36     9.99 negligible  negligible  TRUE        \n",
       "3 ericsson     25.19   -11.36 negligible  negligible FALSE        \n",
       "4 naive        22.62     0.00 NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "product = toString(only_arima_worst$Products[1])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'calendar'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'calendar'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'calendar'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\"  \"calendar\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>18.97       </td><td>-27.74      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>14.86       </td><td> -0.07      </td><td>small       </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>13.75       </td><td>  7.41      </td><td>small       </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>14.85       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 18.97          & -27.74         & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t ericsson     & 14.86        &  -0.07       & small        & negligible   &  TRUE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 13.75          &   7.41         & small          & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t naive        & 14.85        &   0.00       & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 18.97        | -27.74       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ericsson     | 14.86        |  -0.07       | small        | negligible   |  TRUE        | TRUE         | \n",
       "| ets_th:all   | 13.75        |   7.41       | small        | negligible   |  TRUE        | TRUE         | \n",
       "| naive        | 14.85        |   0.00       | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 18.97   -27.74 negligible  negligible FALSE        \n",
       "2 ericsson     14.86    -0.07 small       negligible  TRUE        \n",
       "3 ets_th:all   13.75     7.41 small       negligible  TRUE        \n",
       "4 naive        14.85     0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>7.92        </td><td>19.59       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>7.46        </td><td>24.26       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>7.78        </td><td>21.02       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>9.85        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 7.92           & 19.59          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 7.46           & 24.26          & small          & small          &  TRUE         \\\\\n",
       "\t ericsson     & 7.78         & 21.02        & negligible   & negligible   &  TRUE       \\\\\n",
       "\t naive        & 9.85         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 7.92         | 19.59        | negligible   | negligible   |  TRUE        | \n",
       "| ets_th:all   | 7.46         | 24.26        | small        | small        |  TRUE        | \n",
       "| ericsson     | 7.78         | 21.02        | negligible   | negligible   |  TRUE        | \n",
       "| naive        | 9.85         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 7.92    19.59  negligible  negligible  TRUE        \n",
       "2 ets_th:all   7.46    24.26  small       small       TRUE        \n",
       "3 ericsson     7.78    21.02  negligible  negligible  TRUE        \n",
       "4 naive        9.85     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "product = toString(only_arima_worst$Products[2])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'open_office'</li>\n",
       "\t<li>'calc'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'open\\_office'\n",
       "\\item 'calc'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'open_office'\n",
       "2. 'calc'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"open_office\" \"calc\"       "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>17.21       </td><td>-24.35      </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>19.18       </td><td>-38.58      </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>11.70       </td><td> 15.46      </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>13.84       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 17.21          & -24.35         & small          & negligible     & TRUE           & TRUE          \\\\\n",
       "\t ericsson     & 19.18        & -38.58       & small        & negligible   & TRUE         & TRUE        \\\\\n",
       "\t ets\\_th:all   & 11.70          &  15.46         & small          & negligible     & TRUE           & TRUE          \\\\\n",
       "\t naive        & 13.84        &   0.00       & negligible   & negligible   &   NA         & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 17.21        | -24.35       | small        | negligible   | TRUE         | TRUE         | \n",
       "| ericsson     | 19.18        | -38.58       | small        | negligible   | TRUE         | TRUE         | \n",
       "| ets_th:all   | 11.70        |  15.46       | small        | negligible   | TRUE         | TRUE         | \n",
       "| naive        | 13.84        |   0.00       | negligible   | negligible   |   NA         | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 17.21   -24.35 small       negligible TRUE         \n",
       "2 ericsson     19.18   -38.58 small       negligible TRUE         \n",
       "3 ets_th:all   11.70    15.46 small       negligible TRUE         \n",
       "4 naive        13.84     0.00 negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td> 5.74       </td><td>46.30       </td><td>small       </td><td>medium      </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td> 5.67       </td><td>46.96       </td><td>small       </td><td>medium      </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td> 7.77       </td><td>27.32       </td><td>small       </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>10.69       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all &  5.74          & 46.30          & small          & medium         &  TRUE         \\\\\n",
       "\t ets\\_th:all   &  5.67          & 46.96          & small          & medium         &  TRUE         \\\\\n",
       "\t ericsson     &  7.77        & 27.32        & small        & negligible   &  TRUE       \\\\\n",
       "\t naive        & 10.69        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all |  5.74        | 46.30        | small        | medium       |  TRUE        | \n",
       "| ets_th:all   |  5.67        | 46.96        | small        | medium       |  TRUE        | \n",
       "| ericsson     |  7.77        | 27.32        | small        | negligible   |  TRUE        | \n",
       "| naive        | 10.69        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all  5.74   46.30  small       medium      TRUE        \n",
       "2 ets_th:all    5.67   46.96  small       medium      TRUE        \n",
       "3 ericsson      7.77   27.32  small       negligible  TRUE        \n",
       "4 naive        10.69    0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "product = toString(only_arima_worst$Products[3])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Well predicted backlogs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Products</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>apache_ant         </td></tr>\n",
       "\t<tr><td>apache_apache2     </td></tr>\n",
       "\t<tr><td>eclipse_birt       </td></tr>\n",
       "\t<tr><td>eclipse_data_tools </td></tr>\n",
       "\t<tr><td>eclipse_pde        </td></tr>\n",
       "\t<tr><td>eclipse_platform   </td></tr>\n",
       "\t<tr><td>kernel_file_system </td></tr>\n",
       "\t<tr><td>libre_office_calc  </td></tr>\n",
       "\t<tr><td>libre_office_draw  </td></tr>\n",
       "\t<tr><td>libre_office_writer</td></tr>\n",
       "\t<tr><td>mozilla_core       </td></tr>\n",
       "\t<tr><td>mozilla_firefox    </td></tr>\n",
       "\t<tr><td>mozilla_thunderbird</td></tr>\n",
       "\t<tr><td>open_office_writer </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|l}\n",
       " Products\\\\\n",
       "\\hline\n",
       "\t apache\\_ant         \\\\\n",
       "\t apache\\_apache2     \\\\\n",
       "\t eclipse\\_birt       \\\\\n",
       "\t eclipse\\_data\\_tools \\\\\n",
       "\t eclipse\\_pde        \\\\\n",
       "\t eclipse\\_platform   \\\\\n",
       "\t kernel\\_file\\_system \\\\\n",
       "\t libre\\_office\\_calc  \\\\\n",
       "\t libre\\_office\\_draw  \\\\\n",
       "\t libre\\_office\\_writer\\\\\n",
       "\t mozilla\\_core       \\\\\n",
       "\t mozilla\\_firefox    \\\\\n",
       "\t mozilla\\_thunderbird\\\\\n",
       "\t open\\_office\\_writer \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Products | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| apache_ant          | \n",
       "| apache_apache2      | \n",
       "| eclipse_birt        | \n",
       "| eclipse_data_tools  | \n",
       "| eclipse_pde         | \n",
       "| eclipse_platform    | \n",
       "| kernel_file_system  | \n",
       "| libre_office_calc   | \n",
       "| libre_office_draw   | \n",
       "| libre_office_writer | \n",
       "| mozilla_core        | \n",
       "| mozilla_firefox     | \n",
       "| mozilla_thunderbird | \n",
       "| open_office_writer  | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "   Products           \n",
       "1  apache_ant         \n",
       "2  apache_apache2     \n",
       "3  eclipse_birt       \n",
       "4  eclipse_data_tools \n",
       "5  eclipse_pde        \n",
       "6  eclipse_platform   \n",
       "7  kernel_file_system \n",
       "8  libre_office_calc  \n",
       "9  libre_office_draw  \n",
       "10 libre_office_writer\n",
       "11 mozilla_core       \n",
       "12 mozilla_firefox    \n",
       "13 mozilla_thunderbird\n",
       "14 open_office_writer "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models_better_than_naiv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'apache'</li>\n",
       "\t<li>'ant'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'apache'\n",
       "\\item 'ant'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'apache'\n",
       "2. 'ant'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"apache\" \"ant\"   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>20.12       </td><td>11.33       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>19.79       </td><td>12.78       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>20.43       </td><td> 9.96       </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>22.69       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 20.12          & 11.33          & small          & negligible     & TRUE           & TRUE          \\\\\n",
       "\t ericsson     & 19.79        & 12.78        & small        & negligible   & TRUE         & TRUE        \\\\\n",
       "\t ets\\_th:all   & 20.43          &  9.96          & negligible     & negligible     & TRUE           & TRUE          \\\\\n",
       "\t naive        & 22.69        &  0.00        & negligible   & negligible   &   NA         & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 20.12        | 11.33        | small        | negligible   | TRUE         | TRUE         | \n",
       "| ericsson     | 19.79        | 12.78        | small        | negligible   | TRUE         | TRUE         | \n",
       "| ets_th:all   | 20.43        |  9.96        | negligible   | negligible   | TRUE         | TRUE         | \n",
       "| naive        | 22.69        |  0.00        | negligible   | negligible   |   NA         | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 20.12   11.33  small       negligible TRUE         \n",
       "2 ericsson     19.79   12.78  small       negligible TRUE         \n",
       "3 ets_th:all   20.43    9.96  negligible  negligible TRUE         \n",
       "4 naive        22.69    0.00  negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>3.35        </td><td>46.57       </td><td>negligible  </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>3.01        </td><td>51.99       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.42        </td><td>45.45       </td><td>negligible  </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>6.27        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 3.35           & 46.57          & negligible     & small          &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 3.01           & 51.99          & small          & small          &  TRUE         \\\\\n",
       "\t ericsson     & 3.42         & 45.45        & negligible   & small        &  TRUE       \\\\\n",
       "\t naive        & 6.27         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 3.35         | 46.57        | negligible   | small        |  TRUE        | \n",
       "| ets_th:all   | 3.01         | 51.99        | small        | small        |  TRUE        | \n",
       "| ericsson     | 3.42         | 45.45        | negligible   | small        |  TRUE        | \n",
       "| naive        | 6.27         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g Wilcoxon_test\n",
       "1 arima_th:all 3.35    46.57  negligible  small     TRUE        \n",
       "2 ets_th:all   3.01    51.99  small       small     TRUE        \n",
       "3 ericsson     3.42    45.45  negligible  small     TRUE        \n",
       "4 naive        6.27     0.00  NA          NA       FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[1])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'apache'</li>\n",
       "\t<li>'apache2'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'apache'\n",
       "\\item 'apache2'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'apache'\n",
       "2. 'apache2'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"apache\"  \"apache2\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td> 6.13       </td><td> 20.49      </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>10.49       </td><td>-36.06      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td> 7.36       </td><td>  4.54      </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td> 7.71       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all &  6.13          &  20.49         & negligible     & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t ericsson     & 10.49        & -36.06       & negligible   & negligible   & FALSE        & TRUE        \\\\\n",
       "\t ets\\_th:all   &  7.36          &   4.54         & negligible     & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t naive        &  7.71        &   0.00       & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all |  6.13        |  20.49       | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| ericsson     | 10.49        | -36.06       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ets_th:all   |  7.36        |   4.54       | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| naive        |  7.71        |   0.00       | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all  6.13    20.49 negligible  negligible  TRUE        \n",
       "2 ericsson     10.49   -36.06 negligible  negligible FALSE        \n",
       "3 ets_th:all    7.36     4.54 negligible  negligible  TRUE        \n",
       "4 naive         7.71     0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>4.53        </td><td>27.29       </td><td>negligible  </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>4.59        </td><td>26.32       </td><td>negligible  </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>5.82        </td><td> 6.58       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>6.23        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 4.53           & 27.29          & negligible     & small          &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 4.59           & 26.32          & negligible     & small          &  TRUE         \\\\\n",
       "\t ericsson     & 5.82         &  6.58        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 6.23         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 4.53         | 27.29        | negligible   | small        |  TRUE        | \n",
       "| ets_th:all   | 4.59         | 26.32        | negligible   | small        |  TRUE        | \n",
       "| ericsson     | 5.82         |  6.58        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 6.23         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 4.53    27.29  negligible  small       TRUE        \n",
       "2 ets_th:all   4.59    26.32  negligible  small       TRUE        \n",
       "3 ericsson     5.82     6.58  negligible  negligible FALSE        \n",
       "4 naive        6.23     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[2])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'birt'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'birt'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'birt'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\" \"birt\"   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>23.51       </td><td> 2.65       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>23.61       </td><td> 2.24       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>20.67       </td><td>14.41       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>24.15       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 23.51          &  2.65          & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t ericsson     & 23.61        &  2.24        & negligible   & negligible   & FALSE        & FALSE       \\\\\n",
       "\t ets\\_th:all   & 20.67          & 14.41          & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t naive        & 24.15        &  0.00        & negligible   & negligible   &    NA        & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 23.51        |  2.65        | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ericsson     | 23.61        |  2.24        | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ets_th:all   | 20.67        | 14.41        | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| naive        | 24.15        |  0.00        | negligible   | negligible   |    NA        | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 23.51    2.65  negligible  negligible FALSE        \n",
       "2 ericsson     23.61    2.24  negligible  negligible FALSE        \n",
       "3 ets_th:all   20.67   14.41  negligible  negligible FALSE        \n",
       "4 naive        24.15    0.00  negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>15.17       </td><td> 7.27       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>14.82       </td><td> 9.41       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>17.74       </td><td>-8.44       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>16.36       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 15.17          &  7.27          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 14.82          &  9.41          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ericsson     & 17.74        & -8.44        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 16.36        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 15.17        |  7.27        | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 14.82        |  9.41        | negligible   | negligible   |  TRUE        | \n",
       "| ericsson     | 17.74        | -8.44        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 16.36        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 15.17    7.27  negligible  negligible FALSE        \n",
       "2 ets_th:all   14.82    9.41  negligible  negligible  TRUE        \n",
       "3 ericsson     17.74   -8.44  negligible  negligible FALSE        \n",
       "4 naive        16.36    0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[3])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'data_tools'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'data\\_tools'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'data_tools'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\"    \"data_tools\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>8.12        </td><td>7.20        </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>8.52        </td><td>2.63        </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>8.37        </td><td>4.34        </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>8.75        </td><td>0.00        </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 8.12           & 7.20           & negligible     & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t ericsson     & 8.52         & 2.63         & negligible   & negligible   & FALSE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 8.37           & 4.34           & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t naive        & 8.75         & 0.00         & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 8.12         | 7.20         | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| ericsson     | 8.52         | 2.63         | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ets_th:all   | 8.37         | 4.34         | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| naive        | 8.75         | 0.00         | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 8.12    7.20   negligible  negligible  TRUE        \n",
       "2 ericsson     8.52    2.63   negligible  negligible FALSE        \n",
       "3 ets_th:all   8.37    4.34   negligible  negligible FALSE        \n",
       "4 naive        8.75    0.00   negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>3.43        </td><td> 5.51       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>3.63        </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.23        </td><td>11.02       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>3.63        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 3.43           &  5.51          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 3.63           &  0.00          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ericsson     & 3.23         & 11.02        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 3.63         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 3.43         |  5.51        | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 3.63         |  0.00        | negligible   | negligible   | FALSE        | \n",
       "| ericsson     | 3.23         | 11.02        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 3.63         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 3.43     5.51  negligible  negligible FALSE        \n",
       "2 ets_th:all   3.63     0.00  negligible  negligible FALSE        \n",
       "3 ericsson     3.23    11.02  negligible  negligible FALSE        \n",
       "4 naive        3.63     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[4])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'pde'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'pde'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'pde'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\" \"pde\"    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>8.07        </td><td>  3.35      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>9.67        </td><td>-15.81      </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>8.24        </td><td>  1.32      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>8.35        </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 8.07           &   3.35         & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t ericsson     & 9.67         & -15.81       & negligible   & negligible   &  TRUE        & FALSE       \\\\\n",
       "\t ets\\_th:all   & 8.24           &   1.32         & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t naive        & 8.35         &   0.00       & negligible   & negligible   &    NA        & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 8.07         |   3.35       | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ericsson     | 9.67         | -15.81       | negligible   | negligible   |  TRUE        | FALSE        | \n",
       "| ets_th:all   | 8.24         |   1.32       | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| naive        | 8.35         |   0.00       | negligible   | negligible   |    NA        | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 8.07      3.35 negligible  negligible FALSE        \n",
       "2 ericsson     9.67    -15.81 negligible  negligible  TRUE        \n",
       "3 ets_th:all   8.24      1.32 negligible  negligible FALSE        \n",
       "4 naive        8.35      0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>7.61        </td><td> 3.43       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>7.60        </td><td> 3.55       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>8.63        </td><td>-9.52       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>7.88        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 7.61           &  3.43          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 7.60           &  3.55          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ericsson     & 8.63         & -9.52        & negligible   & negligible   &  TRUE       \\\\\n",
       "\t naive        & 7.88         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 7.61         |  3.43        | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 7.60         |  3.55        | negligible   | negligible   | FALSE        | \n",
       "| ericsson     | 8.63         | -9.52        | negligible   | negligible   |  TRUE        | \n",
       "| naive        | 7.88         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 7.61     3.43  negligible  negligible FALSE        \n",
       "2 ets_th:all   7.60     3.55  negligible  negligible FALSE        \n",
       "3 ericsson     8.63    -9.52  negligible  negligible  TRUE        \n",
       "4 naive        7.88     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[5])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'platform'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'platform'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'platform'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\"  \"platform\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>45.43       </td><td> 7.49       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>46.65       </td><td> 5.01       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>44.05       </td><td>10.30       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>49.11       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 45.43          &  7.49          & negligible     & negligible     & FALSE          & FALSE         \\\\\n",
       "\t ericsson     & 46.65        &  5.01        & negligible   & negligible   & FALSE        & FALSE       \\\\\n",
       "\t ets\\_th:all   & 44.05          & 10.30          & negligible     & negligible     &  TRUE          & FALSE         \\\\\n",
       "\t naive        & 49.11        &  0.00        & negligible   & negligible   &    NA        & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 45.43        |  7.49        | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ericsson     | 46.65        |  5.01        | negligible   | negligible   | FALSE        | FALSE        | \n",
       "| ets_th:all   | 44.05        | 10.30        | negligible   | negligible   |  TRUE        | FALSE        | \n",
       "| naive        | 49.11        |  0.00        | negligible   | negligible   |    NA        | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 45.43    7.49  negligible  negligible FALSE        \n",
       "2 ericsson     46.65    5.01  negligible  negligible FALSE        \n",
       "3 ets_th:all   44.05   10.30  negligible  negligible  TRUE        \n",
       "4 naive        49.11    0.00  negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>44.99       </td><td> 5.48       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>42.50       </td><td>10.71       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>43.39       </td><td> 8.84       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>47.60       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 44.99          &  5.48          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 42.50          & 10.71          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ericsson     & 43.39        &  8.84        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 47.60        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 44.99        |  5.48        | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 42.50        | 10.71        | negligible   | negligible   |  TRUE        | \n",
       "| ericsson     | 43.39        |  8.84        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 47.60        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 44.99    5.48  negligible  negligible FALSE        \n",
       "2 ets_th:all   42.50   10.71  negligible  negligible  TRUE        \n",
       "3 ericsson     43.39    8.84  negligible  negligible FALSE        \n",
       "4 naive        47.60    0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[6])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'kernel'</li>\n",
       "\t<li>'file_system'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'kernel'\n",
       "\\item 'file\\_system'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'kernel'\n",
       "2. 'file_system'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"kernel\"      \"file_system\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>2.68        </td><td>  5.30      </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.50        </td><td>-23.67      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>2.74        </td><td>  3.18      </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>2.83        </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.68           &   5.30         & negligible     & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t ericsson     & 3.50         & -23.67       & negligible   & negligible   & FALSE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 2.74           &   3.18         & negligible     & negligible     & FALSE          & TRUE          \\\\\n",
       "\t naive        & 2.83         &   0.00       & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 2.68         |   5.30       | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| ericsson     | 3.50         | -23.67       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ets_th:all   | 2.74         |   3.18       | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| naive        | 2.83         |   0.00       | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 2.68      5.30 negligible  negligible  TRUE        \n",
       "2 ericsson     3.50    -23.67 negligible  negligible FALSE        \n",
       "3 ets_th:all   2.74      3.18 negligible  negligible FALSE        \n",
       "4 naive        2.83      0.00 negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>2.95        </td><td>-5.73       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>2.88        </td><td>-3.23       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>2.86        </td><td>-2.51       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>2.79        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.95           & -5.73          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ets\\_th:all   & 2.88           & -3.23          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ericsson     & 2.86         & -2.51        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 2.79         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 2.95         | -5.73        | negligible   | negligible   | FALSE        | \n",
       "| ets_th:all   | 2.88         | -3.23        | negligible   | negligible   | FALSE        | \n",
       "| ericsson     | 2.86         | -2.51        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 2.79         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 2.95    -5.73  negligible  negligible FALSE        \n",
       "2 ets_th:all   2.88    -3.23  negligible  negligible FALSE        \n",
       "3 ericsson     2.86    -2.51  negligible  negligible FALSE        \n",
       "4 naive        2.79     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[7])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'libre_office'</li>\n",
       "\t<li>'calc'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'libre\\_office'\n",
       "\\item 'calc'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'libre_office'\n",
       "2. 'calc'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"libre_office\" \"calc\"        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>11.38       </td><td>35.41       </td><td>medium      </td><td>small       </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>12.69       </td><td>27.98       </td><td>medium      </td><td>small       </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>12.01       </td><td>31.84       </td><td>medium      </td><td>small       </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>17.62       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 11.38          & 35.41          & medium         & small          & TRUE           & TRUE          \\\\\n",
       "\t ericsson     & 12.69        & 27.98        & medium       & small        & TRUE         & TRUE        \\\\\n",
       "\t ets\\_th:all   & 12.01          & 31.84          & medium         & small          & TRUE           & TRUE          \\\\\n",
       "\t naive        & 17.62        &  0.00        & negligible   & negligible   &   NA         & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 11.38        | 35.41        | medium       | small        | TRUE         | TRUE         | \n",
       "| ericsson     | 12.69        | 27.98        | medium       | small        | TRUE         | TRUE         | \n",
       "| ets_th:all   | 12.01        | 31.84        | medium       | small        | TRUE         | TRUE         | \n",
       "| naive        | 17.62        |  0.00        | negligible   | negligible   |   NA         | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 11.38   35.41  medium      small      TRUE         \n",
       "2 ericsson     12.69   27.98  medium      small      TRUE         \n",
       "3 ets_th:all   12.01   31.84  medium      small      TRUE         \n",
       "4 naive        17.62    0.00  negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>13.31       </td><td>24.25       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>12.94       </td><td>26.35       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>12.10       </td><td>31.13       </td><td>medium      </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>17.57       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 13.31          & 24.25          & small          & small          &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 12.94          & 26.35          & small          & small          &  TRUE         \\\\\n",
       "\t ericsson     & 12.10        & 31.13        & medium       & small        &  TRUE       \\\\\n",
       "\t naive        & 17.57        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 13.31        | 24.25        | small        | small        |  TRUE        | \n",
       "| ets_th:all   | 12.94        | 26.35        | small        | small        |  TRUE        | \n",
       "| ericsson     | 12.10        | 31.13        | medium       | small        |  TRUE        | \n",
       "| naive        | 17.57        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g Wilcoxon_test\n",
       "1 arima_th:all 13.31   24.25  small       small     TRUE        \n",
       "2 ets_th:all   12.94   26.35  small       small     TRUE        \n",
       "3 ericsson     12.10   31.13  medium      small     TRUE        \n",
       "4 naive        17.57    0.00  NA          NA       FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[8])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'libre_office'</li>\n",
       "\t<li>'draw'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'libre\\_office'\n",
       "\\item 'draw'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'libre_office'\n",
       "2. 'draw'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"libre_office\" \"draw\"        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>3.46        </td><td>11.51       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.43        </td><td>12.28       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>3.51        </td><td>10.23       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>3.91        </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 3.46           & 11.51          & negligible     & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t ericsson     & 3.43         & 12.28        & negligible   & negligible   & FALSE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 3.51           & 10.23          & negligible     & negligible     &  TRUE          & TRUE          \\\\\n",
       "\t naive        & 3.91         &  0.00        & negligible   & negligible   &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 3.46         | 11.51        | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| ericsson     | 3.43         | 12.28        | negligible   | negligible   | FALSE        | TRUE         | \n",
       "| ets_th:all   | 3.51         | 10.23        | negligible   | negligible   |  TRUE        | TRUE         | \n",
       "| naive        | 3.91         |  0.00        | negligible   | negligible   |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 3.46    11.51  negligible  negligible  TRUE        \n",
       "2 ericsson     3.43    12.28  negligible  negligible FALSE        \n",
       "3 ets_th:all   3.51    10.23  negligible  negligible  TRUE        \n",
       "4 naive        3.91     0.00  negligible  negligible    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>3.24        </td><td>14.96       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>3.63        </td><td> 4.72       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.32        </td><td>12.86       </td><td>negligible  </td><td>negligible  </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>3.81        </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 3.24           & 14.96          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 3.63           &  4.72          & negligible     & negligible     & FALSE         \\\\\n",
       "\t ericsson     & 3.32         & 12.86        & negligible   & negligible   & FALSE       \\\\\n",
       "\t naive        & 3.81         &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 3.24         | 14.96        | negligible   | negligible   |  TRUE        | \n",
       "| ets_th:all   | 3.63         |  4.72        | negligible   | negligible   | FALSE        | \n",
       "| ericsson     | 3.32         | 12.86        | negligible   | negligible   | FALSE        | \n",
       "| naive        | 3.81         |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 3.24    14.96  negligible  negligible  TRUE        \n",
       "2 ets_th:all   3.63     4.72  negligible  negligible FALSE        \n",
       "3 ericsson     3.32    12.86  negligible  negligible FALSE        \n",
       "4 naive        3.81     0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[9])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'libre_office'</li>\n",
       "\t<li>'writer'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'libre\\_office'\n",
       "\\item 'writer'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'libre_office'\n",
       "2. 'writer'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"libre_office\" \"writer\"      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>20.53       </td><td>28.69       </td><td>small       </td><td>small       </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>24.19       </td><td>15.98       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>20.90       </td><td>27.41       </td><td>small       </td><td>small       </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>28.79       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 20.53          & 28.69          & small          & small          & TRUE           & TRUE          \\\\\n",
       "\t ericsson     & 24.19        & 15.98        & small        & negligible   & TRUE         & TRUE        \\\\\n",
       "\t ets\\_th:all   & 20.90          & 27.41          & small          & small          & TRUE           & TRUE          \\\\\n",
       "\t naive        & 28.79        &  0.00        & negligible   & negligible   &   NA         & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 20.53        | 28.69        | small        | small        | TRUE         | TRUE         | \n",
       "| ericsson     | 24.19        | 15.98        | small        | negligible   | TRUE         | TRUE         | \n",
       "| ets_th:all   | 20.90        | 27.41        | small        | small        | TRUE         | TRUE         | \n",
       "| naive        | 28.79        |  0.00        | negligible   | negligible   |   NA         | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 20.53   28.69  small       small      TRUE         \n",
       "2 ericsson     24.19   15.98  small       negligible TRUE         \n",
       "3 ets_th:all   20.90   27.41  small       small      TRUE         \n",
       "4 naive        28.79    0.00  negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>20.88       </td><td>25.35       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>21.73       </td><td>22.31       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>20.76       </td><td>25.78       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>27.97       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 20.88          & 25.35          & small          & small          &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 21.73          & 22.31          & small          & small          &  TRUE         \\\\\n",
       "\t ericsson     & 20.76        & 25.78        & small        & small        &  TRUE       \\\\\n",
       "\t naive        & 27.97        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 20.88        | 25.35        | small        | small        |  TRUE        | \n",
       "| ets_th:all   | 21.73        | 22.31        | small        | small        |  TRUE        | \n",
       "| ericsson     | 20.76        | 25.78        | small        | small        |  TRUE        | \n",
       "| naive        | 27.97        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g Wilcoxon_test\n",
       "1 arima_th:all 20.88   25.35  small       small     TRUE        \n",
       "2 ets_th:all   21.73   22.31  small       small     TRUE        \n",
       "3 ericsson     20.76   25.78  small       small     TRUE        \n",
       "4 naive        27.97    0.00  NA          NA       FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[10])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'core'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'core'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'core'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\" \"core\"   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>117.35      </td><td>18.77       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>131.98      </td><td> 8.65       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>125.02      </td><td>13.46       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>144.47      </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 117.35         & 18.77          & small          & negligible     & TRUE           & TRUE          \\\\\n",
       "\t ericsson     & 131.98       &  8.65        & small        & negligible   & TRUE         & TRUE        \\\\\n",
       "\t ets\\_th:all   & 125.02         & 13.46          & small          & negligible     & TRUE           & TRUE          \\\\\n",
       "\t naive        & 144.47       &  0.00        & negligible   & negligible   &   NA         & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 117.35       | 18.77        | small        | negligible   | TRUE         | TRUE         | \n",
       "| ericsson     | 131.98       |  8.65        | small        | negligible   | TRUE         | TRUE         | \n",
       "| ets_th:all   | 125.02       | 13.46        | small        | negligible   | TRUE         | TRUE         | \n",
       "| naive        | 144.47       |  0.00        | negligible   | negligible   |   NA         | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 117.35  18.77  small       negligible TRUE         \n",
       "2 ericsson     131.98   8.65  small       negligible TRUE         \n",
       "3 ets_th:all   125.02  13.46  small       negligible TRUE         \n",
       "4 naive        144.47   0.00  negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td> 87.04      </td><td>21.32       </td><td>small       </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td> 84.59      </td><td>23.54       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td> 85.89      </td><td>22.36       </td><td>small       </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>110.63      </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all &  87.04         & 21.32          & small          & negligible     &  TRUE         \\\\\n",
       "\t ets\\_th:all   &  84.59         & 23.54          & small          & small          &  TRUE         \\\\\n",
       "\t ericsson     &  85.89       & 22.36        & small        & negligible   &  TRUE       \\\\\n",
       "\t naive        & 110.63       &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all |  87.04       | 21.32        | small        | negligible   |  TRUE        | \n",
       "| ets_th:all   |  84.59       | 23.54        | small        | small        |  TRUE        | \n",
       "| ericsson     |  85.89       | 22.36        | small        | negligible   |  TRUE        | \n",
       "| naive        | 110.63       |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all  87.04  21.32  small       negligible  TRUE        \n",
       "2 ets_th:all    84.59  23.54  small       small       TRUE        \n",
       "3 ericsson      85.89  22.36  small       negligible  TRUE        \n",
       "4 naive        110.63   0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[11])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'firefox'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'firefox'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'firefox'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\" \"firefox\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>60.17       </td><td>14.19       </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>71.03       </td><td>-1.30       </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>58.61       </td><td>16.41       </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>70.12       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 60.17          & 14.19          & negligible     & negligible     & TRUE           & FALSE         \\\\\n",
       "\t ericsson     & 71.03        & -1.30        & small        & negligible   & TRUE         & FALSE       \\\\\n",
       "\t ets\\_th:all   & 58.61          & 16.41          & negligible     & negligible     & TRUE           & FALSE         \\\\\n",
       "\t naive        & 70.12        &  0.00        & negligible   & negligible   &   NA         & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 60.17        | 14.19        | negligible   | negligible   | TRUE         | FALSE        | \n",
       "| ericsson     | 71.03        | -1.30        | small        | negligible   | TRUE         | FALSE        | \n",
       "| ets_th:all   | 58.61        | 16.41        | negligible   | negligible   | TRUE         | FALSE        | \n",
       "| naive        | 70.12        |  0.00        | negligible   | negligible   |   NA         | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 60.17   14.19  negligible  negligible TRUE         \n",
       "2 ericsson     71.03   -1.30  small       negligible TRUE         \n",
       "3 ets_th:all   58.61   16.41  negligible  negligible TRUE         \n",
       "4 naive        70.12    0.00  negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>56.81       </td><td>18.97       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>56.45       </td><td>19.48       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>57.44       </td><td>18.07       </td><td>small       </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>70.11       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 56.81          & 18.97          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 56.45          & 19.48          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ericsson     & 57.44        & 18.07        & small        & negligible   &  TRUE       \\\\\n",
       "\t naive        & 70.11        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 56.81        | 18.97        | negligible   | negligible   |  TRUE        | \n",
       "| ets_th:all   | 56.45        | 19.48        | negligible   | negligible   |  TRUE        | \n",
       "| ericsson     | 57.44        | 18.07        | small        | negligible   |  TRUE        | \n",
       "| naive        | 70.11        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 56.81   18.97  negligible  negligible  TRUE        \n",
       "2 ets_th:all   56.45   19.48  negligible  negligible  TRUE        \n",
       "3 ericsson     57.44   18.07  small       negligible  TRUE        \n",
       "4 naive        70.11    0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[12])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'thunderbird'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'thunderbird'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'thunderbird'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\"     \"thunderbird\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>18.70       </td><td>15.15       </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>20.94       </td><td> 4.99       </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>19.52       </td><td>11.43       </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>22.04       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 18.70          & 15.15          & negligible     & negligible     & TRUE           & FALSE         \\\\\n",
       "\t ericsson     & 20.94        &  4.99        & negligible   & negligible   & TRUE         & FALSE       \\\\\n",
       "\t ets\\_th:all   & 19.52          & 11.43          & negligible     & negligible     & TRUE           & FALSE         \\\\\n",
       "\t naive        & 22.04        &  0.00        & negligible   & negligible   &   NA         & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 18.70        | 15.15        | negligible   | negligible   | TRUE         | FALSE        | \n",
       "| ericsson     | 20.94        |  4.99        | negligible   | negligible   | TRUE         | FALSE        | \n",
       "| ets_th:all   | 19.52        | 11.43        | negligible   | negligible   | TRUE         | FALSE        | \n",
       "| naive        | 22.04        |  0.00        | negligible   | negligible   |   NA         | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 18.70   15.15  negligible  negligible TRUE         \n",
       "2 ericsson     20.94    4.99  negligible  negligible TRUE         \n",
       "3 ets_th:all   19.52   11.43  negligible  negligible TRUE         \n",
       "4 naive        22.04    0.00  negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>17.40       </td><td>17.34       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>18.00       </td><td>14.49       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>17.62       </td><td>16.29       </td><td>negligible  </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>21.05       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 17.40          & 17.34          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 18.00          & 14.49          & negligible     & negligible     &  TRUE         \\\\\n",
       "\t ericsson     & 17.62        & 16.29        & negligible   & negligible   &  TRUE       \\\\\n",
       "\t naive        & 21.05        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 17.40        | 17.34        | negligible   | negligible   |  TRUE        | \n",
       "| ets_th:all   | 18.00        | 14.49        | negligible   | negligible   |  TRUE        | \n",
       "| ericsson     | 17.62        | 16.29        | negligible   | negligible   |  TRUE        | \n",
       "| naive        | 21.05        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 17.40   17.34  negligible  negligible  TRUE        \n",
       "2 ets_th:all   18.00   14.49  negligible  negligible  TRUE        \n",
       "3 ericsson     17.62   16.29  negligible  negligible  TRUE        \n",
       "4 naive        21.05    0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[13])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'open_office'</li>\n",
       "\t<li>'writer'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'open\\_office'\n",
       "\\item 'writer'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'open_office'\n",
       "2. 'writer'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"open_office\" \"writer\"     "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>46.25       </td><td>  0.02      </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>54.39       </td><td>-17.57      </td><td>small       </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>44.70       </td><td>  3.37      </td><td>negligible  </td><td>negligible  </td><td>TRUE        </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>46.26       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>  NA        </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 46.25          &   0.02         & negligible     & negligible     & TRUE           & TRUE          \\\\\n",
       "\t ericsson     & 54.39        & -17.57       & small        & negligible   & TRUE         & TRUE        \\\\\n",
       "\t ets\\_th:all   & 44.70          &   3.37         & negligible     & negligible     & TRUE           & TRUE          \\\\\n",
       "\t naive        & 46.26        &   0.00       & negligible   & negligible   &   NA         & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 46.25        |   0.02       | negligible   | negligible   | TRUE         | TRUE         | \n",
       "| ericsson     | 54.39        | -17.57       | small        | negligible   | TRUE         | TRUE         | \n",
       "| ets_th:all   | 44.70        |   3.37       | negligible   | negligible   | TRUE         | TRUE         | \n",
       "| naive        | 46.26        |   0.00       | negligible   | negligible   |   NA         | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 46.25     0.02 negligible  negligible TRUE         \n",
       "2 ericsson     54.39   -17.57 small       negligible TRUE         \n",
       "3 ets_th:all   44.70     3.37 negligible  negligible TRUE         \n",
       "4 naive        46.26     0.00 negligible  negligible   NA         \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "'Divided_backlog_predictions:'"
      ],
      "text/latex": [
       "'Divided\\_backlog\\_predictions:'"
      ],
      "text/markdown": [
       "'Divided_backlog_predictions:'"
      ],
      "text/plain": [
       "[1] \"Divided_backlog_predictions:\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>Wilcoxon_test</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>11.73       </td><td>50.30       </td><td>small       </td><td>small       </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>10.54       </td><td>55.34       </td><td>small       </td><td>medium      </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>14.65       </td><td>37.92       </td><td>small       </td><td>negligible  </td><td> TRUE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>23.60       </td><td> 0.00       </td><td>NA          </td><td>NA          </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & Wilcoxon\\_test\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 11.73          & 50.30          & small          & small          &  TRUE         \\\\\n",
       "\t ets\\_th:all   & 10.54          & 55.34          & small          & medium         &  TRUE         \\\\\n",
       "\t ericsson     & 14.65        & 37.92        & small        & negligible   &  TRUE       \\\\\n",
       "\t naive        & 23.60        &  0.00        & NA           & NA           & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | Wilcoxon_test | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 11.73        | 50.30        | small        | small        |  TRUE        | \n",
       "| ets_th:all   | 10.54        | 55.34        | small        | medium       |  TRUE        | \n",
       "| ericsson     | 14.65        | 37.92        | small        | negligible   |  TRUE        | \n",
       "| naive        | 23.60        |  0.00        | NA           | NA           | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   Wilcoxon_test\n",
       "1 arima_th:all 11.73   50.30  small       small       TRUE        \n",
       "2 ets_th:all   10.54   55.34  small       medium      TRUE        \n",
       "3 ericsson     14.65   37.92  small       negligible  TRUE        \n",
       "4 naive        23.60    0.00  NA          NA         FALSE        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "product = toString(models_better_than_naiv$Products[14])\n",
    "summarize_product_predictions(product)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# arima, ets, ericsson > naive "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_methods <- function(products, comparison_baseline_method, cleaned_data){\n",
    "    product_names = c()\n",
    "    arima_better_than_baseline = c()\n",
    "    ets_better_than_baseline = c()\n",
    "    ericsson_better_than_baseline = c()\n",
    "    arima_nnt = c()\n",
    "    ets_nnt = c()\n",
    "    ericsson_nnt = c()\n",
    "    \n",
    "    for(i in 1:nrow(products)){\n",
    "        product = products[i,]\n",
    "        product_names = c(product_names, paste(product[1],\"_\", product[2], sep =\"\"))\n",
    "        if(cleaned_data) {\n",
    "            predictions_evaluation = read.csv(file=paste(\"data/\", product[1], \"/\", product[2], \"/Predictions/Divided_backlog_predictions/\", comparison_baseline_method, \"_baseline_predictions_comparison.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "        } else {\n",
    "            predictions_evaluation = read.csv(file=paste(\"data/\", product[1], \"/\", product[2], \"/\", comparison_baseline_method, \"_baseline_predictions_comparison.csv\", sep=\"\"), header=TRUE, sep=\",\")\n",
    "        }\n",
    "        predictions_evaluation = predictions_evaluation %>% filter (Method == \"arima_th:all\" | Method == \"ericsson\" | Method == \"naive\" | Method == \"ets_th:all\" )\n",
    "        a_nnt = (predictions_evaluation %>% filter (Method == \"arima_th:all\") %>% select (NNT))[[1]]\n",
    "        et_nnt = (predictions_evaluation %>% filter (Method == \"ets_th:all\") %>% select (NNT))[[1]]\n",
    "        er_nnt = (predictions_evaluation %>% filter (Method == \"ericsson\") %>% select (NNT))[[1]]\n",
    "\n",
    "        arima_nnt = c(arima_nnt, a_nnt)\n",
    "        ets_nnt = c(ets_nnt, et_nnt)\n",
    "        ericsson_nnt = c(ericsson_nnt, er_nnt)\n",
    "        \n",
    "        arima_better_than_baseline = c(arima_better_than_baseline, is_method_better(\"arima_th:all\", comparison_baseline_method, predictions_evaluation ))\n",
    "        ets_better_than_baseline = c(ets_better_than_baseline, is_method_better(\"ets_th:all\", comparison_baseline_method, predictions_evaluation ))\n",
    "        ericsson_better_than_baseline = c(ericsson_better_than_baseline, is_method_better(\"ericsson\", comparison_baseline_method, predictions_evaluation ))            \n",
    "        display(product)\n",
    "        display(predictions_evaluation)\n",
    "    }  \n",
    "    summary = data.frame(Products = product_names,\n",
    "                         Arima = arima_better_than_baseline,\n",
    "                         Ets = ets_better_than_baseline,\n",
    "                         Ericsson = ericsson_better_than_baseline\n",
    "               )\n",
    "    arima_better = length(which(arima_better_than_baseline==1))\n",
    "    arima_worst = length(which(arima_better_than_baseline==-1))\n",
    "    \n",
    "    ets_better = length(which(ets_better_than_baseline==1))\n",
    "    ets_worst = length(which(ets_better_than_baseline==-1))\n",
    "    \n",
    "    ericsson_better = length(which(ericsson_better_than_baseline==1))\n",
    "    ericsson_worst = length(which(ericsson_better_than_baseline==-1))\n",
    "\n",
    "#     print(paste(\"Arima mean nnt:\", mean(arima_nnt[arima_nnt>0])))\n",
    "#     print(arima_nnt)\n",
    "#     print(paste(\"ETS mean nnt:\", mean(ets_nnt[ets_nnt>0 & !is.infinite(ets_nnt)])))\n",
    "#     print(ets_nnt)\n",
    "#     print(paste(\"Ericsson mean nnt:\", mean(ericsson_nnt[ericsson_nnt>0 & !is.infinite(ericsson_nnt)])))\n",
    "#     print(ericsson_nnt)\n",
    "\n",
    "    print(paste(\"Arima vs\", comparison_baseline_method))\n",
    "    print(paste(arima_better, \":\", arima_worst))\n",
    "    print(paste(\"ETS vs\", comparison_baseline_method))\n",
    "    print(paste(ets_better, \":\",ets_worst ))\n",
    "    print(paste(\"Ericsson vs\", comparison_baseline_method))\n",
    "    print(paste(ericsson_better, \":\",ericsson_worst ))\n",
    "    display(summary)\n",
    "    if(cleaned_data){\n",
    "         write.table(summary, file =paste(\"data/methods_comparison/cleaned_data/\", comparison_baseline_method, \"_vs_others.csv\", sep=\"\"), sep=\",\")\n",
    "    } else {\n",
    "        write.table(summary, file =paste(\"data/methods_comparison/raw_data/\", comparison_baseline_method, \"_vs_others.csv\", sep=\"\"), sep=\",\")\n",
    "    }\n",
    "}\n",
    "\n",
    "is_method_better <- function(method_1, baseline_method, predictions_evaluation) {\n",
    "    method_1_statistics = predictions_evaluation %>% filter (Method == method_1)\n",
    "    wilcoxon = (method_1_statistics %>% select (Wilcoxon_test))[[1]]\n",
    "    cliff =  (method_1_statistics %>% select (Cliff_delta))[[1]]\n",
    "    hedges =  (method_1_statistics %>% select (Hedges_g))[[1]]\n",
    "    method_1_error =  (method_1_statistics %>% select (Error_1))[[1]]\n",
    "    baseline_method_error = (predictions_evaluation %>% filter (Method == baseline_method) %>% select (Error_1))[[1]]\n",
    "\n",
    "    if(!is.na(wilcoxon) & wilcoxon == TRUE){\n",
    "        if(hedges == \"negligible\" & cliff == \"negligible\"){\n",
    "            is_better = 0\n",
    "        } else {\n",
    "            if(method_1_error<baseline_method_error){\n",
    "                is_better = 1\n",
    "            } else {\n",
    "                is_better = -1\n",
    "            }\n",
    "        }\n",
    "    } else {\n",
    "        is_better = 0\n",
    "    }\n",
    "    return(is_better)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'kernel'</li>\n",
       "\t<li>'networking'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'kernel'\n",
       "\\item 'networking'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'kernel'\n",
       "2. 'networking'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"kernel\"     \"networking\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>1.95         </td><td> -6.56       </td><td>negligible   </td><td>negligible   </td><td>-3.130200e+01</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>2.53         </td><td>-38.25       </td><td>negligible   </td><td>small        </td><td>-5.948000e+00</td><td> TRUE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>1.93         </td><td> -5.46       </td><td>negligible   </td><td>negligible   </td><td>-3.627000e+01</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>1.83         </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td> 5.718313e+17</td><td>   NA        </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 1.95           &  -6.56         & negligible     & negligible     & -3.130200e+01  & FALSE          & TRUE          \\\\\n",
       "\t ericsson      & 2.53          & -38.25        & negligible    & small         & -5.948000e+00 &  TRUE         & TRUE         \\\\\n",
       "\t ets\\_th:all   & 1.93           &  -5.46         & negligible     & negligible     & -3.627000e+01  & FALSE          & TRUE          \\\\\n",
       "\t naive         & 1.83          &   0.00        & negligible    & negligible    &  5.718313e+17 &    NA         & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 1.95          |  -6.56        | negligible    | negligible    | -3.130200e+01 | FALSE         | TRUE          | \n",
       "| ericsson      | 2.53          | -38.25        | negligible    | small         | -5.948000e+00 |  TRUE         | TRUE          | \n",
       "| ets_th:all    | 1.93          |  -5.46        | negligible    | negligible    | -3.627000e+01 | FALSE         | TRUE          | \n",
       "| naive         | 1.83          |   0.00        | negligible    | negligible    |  5.718313e+17 |    NA         | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 1.95     -6.56 negligible  negligible -3.130200e+01\n",
       "2 ericsson     2.53    -38.25 negligible  small      -5.948000e+00\n",
       "3 ets_th:all   1.93     -5.46 negligible  negligible -3.627000e+01\n",
       "4 naive        1.83      0.00 negligible  negligible  5.718313e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         TRUE              \n",
       "2  TRUE         TRUE              \n",
       "3 FALSE         TRUE              \n",
       "4    NA         TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'platform'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'platform'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'platform'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\"  \"platform\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>45.43        </td><td> 7.49        </td><td>negligible   </td><td>negligible   </td><td>-1.473470e+02</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>46.65        </td><td> 5.01        </td><td>negligible   </td><td>negligible   </td><td> 4.709900e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>44.05        </td><td>10.30        </td><td>negligible   </td><td>negligible   </td><td> 3.411300e+01</td><td> TRUE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>naive        </td><td>49.11        </td><td> 0.00        </td><td>negligible   </td><td>negligible   </td><td> 6.666718e+18</td><td>   NA        </td><td>FALSE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 45.43          &  7.49          & negligible     & negligible     & -1.473470e+02  & FALSE          & FALSE         \\\\\n",
       "\t ericsson      & 46.65         &  5.01         & negligible    & negligible    &  4.709900e+01 & FALSE         & FALSE        \\\\\n",
       "\t ets\\_th:all   & 44.05          & 10.30          & negligible     & negligible     &  3.411300e+01  &  TRUE          & FALSE         \\\\\n",
       "\t naive         & 49.11         &  0.00         & negligible    & negligible    &  6.666718e+18 &    NA         & FALSE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 45.43         |  7.49         | negligible    | negligible    | -1.473470e+02 | FALSE         | FALSE         | \n",
       "| ericsson      | 46.65         |  5.01         | negligible    | negligible    |  4.709900e+01 | FALSE         | FALSE         | \n",
       "| ets_th:all    | 44.05         | 10.30         | negligible    | negligible    |  3.411300e+01 |  TRUE         | FALSE         | \n",
       "| naive         | 49.11         |  0.00         | negligible    | negligible    |  6.666718e+18 |    NA         | FALSE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 45.43    7.49  negligible  negligible -1.473470e+02\n",
       "2 ericsson     46.65    5.01  negligible  negligible  4.709900e+01\n",
       "3 ets_th:all   44.05   10.30  negligible  negligible  3.411300e+01\n",
       "4 naive        49.11    0.00  negligible  negligible  6.666718e+18\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         FALSE             \n",
       "2 FALSE         FALSE             \n",
       "3  TRUE         FALSE             \n",
       "4    NA         FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'data_tools'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'data\\_tools'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'data_tools'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\"    \"data_tools\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>8.12         </td><td>7.20         </td><td>negligible   </td><td>negligible   </td><td> 2.283000e+01</td><td> TRUE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>8.52         </td><td>2.63         </td><td>negligible   </td><td>negligible   </td><td>-3.701350e+02</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>8.37         </td><td>4.34         </td><td>negligible   </td><td>negligible   </td><td>-1.722250e+03</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>8.75         </td><td>0.00         </td><td>negligible   </td><td>negligible   </td><td> 8.608826e+16</td><td>   NA        </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 8.12           & 7.20           & negligible     & negligible     &  2.283000e+01  &  TRUE          & TRUE          \\\\\n",
       "\t ericsson      & 8.52          & 2.63          & negligible    & negligible    & -3.701350e+02 & FALSE         & TRUE         \\\\\n",
       "\t ets\\_th:all   & 8.37           & 4.34           & negligible     & negligible     & -1.722250e+03  & FALSE          & TRUE          \\\\\n",
       "\t naive         & 8.75          & 0.00          & negligible    & negligible    &  8.608826e+16 &    NA         & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 8.12          | 7.20          | negligible    | negligible    |  2.283000e+01 |  TRUE         | TRUE          | \n",
       "| ericsson      | 8.52          | 2.63          | negligible    | negligible    | -3.701350e+02 | FALSE         | TRUE          | \n",
       "| ets_th:all    | 8.37          | 4.34          | negligible    | negligible    | -1.722250e+03 | FALSE         | TRUE          | \n",
       "| naive         | 8.75          | 0.00          | negligible    | negligible    |  8.608826e+16 |    NA         | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 8.12    7.20   negligible  negligible  2.283000e+01\n",
       "2 ericsson     8.52    2.63   negligible  negligible -3.701350e+02\n",
       "3 ets_th:all   8.37    4.34   negligible  negligible -1.722250e+03\n",
       "4 naive        8.75    0.00   negligible  negligible  8.608826e+16\n",
       "  Wilcoxon_test Independent_errors\n",
       "1  TRUE         TRUE              \n",
       "2 FALSE         TRUE              \n",
       "3 FALSE         TRUE              \n",
       "4    NA         TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'birt'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'birt'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'birt'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\" \"birt\"   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>23.51        </td><td> 2.65        </td><td>negligible   </td><td>negligible   </td><td>-2.406100e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>23.61        </td><td> 2.24        </td><td>negligible   </td><td>negligible   </td><td>-1.775700e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>20.67        </td><td>14.41        </td><td>negligible   </td><td>negligible   </td><td>-1.254400e+02</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>naive        </td><td>24.15        </td><td> 0.00        </td><td>negligible   </td><td>negligible   </td><td> 9.250904e+17</td><td>   NA        </td><td>FALSE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 23.51          &  2.65          & negligible     & negligible     & -2.406100e+01  & FALSE          & FALSE         \\\\\n",
       "\t ericsson      & 23.61         &  2.24         & negligible    & negligible    & -1.775700e+01 & FALSE         & FALSE        \\\\\n",
       "\t ets\\_th:all   & 20.67          & 14.41          & negligible     & negligible     & -1.254400e+02  & FALSE          & FALSE         \\\\\n",
       "\t naive         & 24.15         &  0.00         & negligible    & negligible    &  9.250904e+17 &    NA         & FALSE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 23.51         |  2.65         | negligible    | negligible    | -2.406100e+01 | FALSE         | FALSE         | \n",
       "| ericsson      | 23.61         |  2.24         | negligible    | negligible    | -1.775700e+01 | FALSE         | FALSE         | \n",
       "| ets_th:all    | 20.67         | 14.41         | negligible    | negligible    | -1.254400e+02 | FALSE         | FALSE         | \n",
       "| naive         | 24.15         |  0.00         | negligible    | negligible    |  9.250904e+17 |    NA         | FALSE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 23.51    2.65  negligible  negligible -2.406100e+01\n",
       "2 ericsson     23.61    2.24  negligible  negligible -1.775700e+01\n",
       "3 ets_th:all   20.67   14.41  negligible  negligible -1.254400e+02\n",
       "4 naive        24.15    0.00  negligible  negligible  9.250904e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         FALSE             \n",
       "2 FALSE         FALSE             \n",
       "3 FALSE         FALSE             \n",
       "4    NA         FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'jdt'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'jdt'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'jdt'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\" \"jdt\"    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>24.57        </td><td> -4.38       </td><td>negligible   </td><td>negligible   </td><td>-1.316000e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>26.01        </td><td>-10.49       </td><td>negligible   </td><td>negligible   </td><td>-1.614000e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>22.17        </td><td>  5.82       </td><td>negligible   </td><td>negligible   </td><td>-1.673790e+02</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>naive        </td><td>23.54        </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td>-8.561605e+18</td><td>   NA        </td><td>FALSE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 24.57          &  -4.38         & negligible     & negligible     & -1.316000e+01  & FALSE          & FALSE         \\\\\n",
       "\t ericsson      & 26.01         & -10.49        & negligible    & negligible    & -1.614000e+01 & FALSE         & FALSE        \\\\\n",
       "\t ets\\_th:all   & 22.17          &   5.82         & negligible     & negligible     & -1.673790e+02  & FALSE          & FALSE         \\\\\n",
       "\t naive         & 23.54         &   0.00        & negligible    & negligible    & -8.561605e+18 &    NA         & FALSE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 24.57         |  -4.38        | negligible    | negligible    | -1.316000e+01 | FALSE         | FALSE         | \n",
       "| ericsson      | 26.01         | -10.49        | negligible    | negligible    | -1.614000e+01 | FALSE         | FALSE         | \n",
       "| ets_th:all    | 22.17         |   5.82        | negligible    | negligible    | -1.673790e+02 | FALSE         | FALSE         | \n",
       "| naive         | 23.54         |   0.00        | negligible    | negligible    | -8.561605e+18 |    NA         | FALSE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 24.57    -4.38 negligible  negligible -1.316000e+01\n",
       "2 ericsson     26.01   -10.49 negligible  negligible -1.614000e+01\n",
       "3 ets_th:all   22.17     5.82 negligible  negligible -1.673790e+02\n",
       "4 naive        23.54     0.00 negligible  negligible -8.561605e+18\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         FALSE             \n",
       "2 FALSE         FALSE             \n",
       "3 FALSE         FALSE             \n",
       "4    NA         FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'eclipse'</li>\n",
       "\t<li>'pde'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'eclipse'\n",
       "\\item 'pde'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'eclipse'\n",
       "2. 'pde'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"eclipse\" \"pde\"    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>8.07         </td><td>  3.35       </td><td>negligible   </td><td>negligible   </td><td> 4.495300e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>9.67         </td><td>-15.81       </td><td>negligible   </td><td>negligible   </td><td>-1.925400e+01</td><td> TRUE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>8.24         </td><td>  1.32       </td><td>negligible   </td><td>negligible   </td><td> 1.342910e+02</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>naive        </td><td>8.35         </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td> 2.196295e+17</td><td>   NA        </td><td>FALSE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 8.07           &   3.35         & negligible     & negligible     &  4.495300e+01  & FALSE          & FALSE         \\\\\n",
       "\t ericsson      & 9.67          & -15.81        & negligible    & negligible    & -1.925400e+01 &  TRUE         & FALSE        \\\\\n",
       "\t ets\\_th:all   & 8.24           &   1.32         & negligible     & negligible     &  1.342910e+02  & FALSE          & FALSE         \\\\\n",
       "\t naive         & 8.35          &   0.00        & negligible    & negligible    &  2.196295e+17 &    NA         & FALSE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 8.07          |   3.35        | negligible    | negligible    |  4.495300e+01 | FALSE         | FALSE         | \n",
       "| ericsson      | 9.67          | -15.81        | negligible    | negligible    | -1.925400e+01 |  TRUE         | FALSE         | \n",
       "| ets_th:all    | 8.24          |   1.32        | negligible    | negligible    |  1.342910e+02 | FALSE         | FALSE         | \n",
       "| naive         | 8.35          |   0.00        | negligible    | negligible    |  2.196295e+17 |    NA         | FALSE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 8.07      3.35 negligible  negligible  4.495300e+01\n",
       "2 ericsson     9.67    -15.81 negligible  negligible -1.925400e+01\n",
       "3 ets_th:all   8.24      1.32 negligible  negligible  1.342910e+02\n",
       "4 naive        8.35      0.00 negligible  negligible  2.196295e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         FALSE             \n",
       "2  TRUE         FALSE             \n",
       "3 FALSE         FALSE             \n",
       "4    NA         FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'calendar'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'calendar'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'calendar'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\"  \"calendar\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>18.97       </td><td>-27.74      </td><td>negligible  </td><td>negligible  </td><td> 1.59100e+01</td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>14.86       </td><td> -0.07      </td><td>small       </td><td>negligible  </td><td> 8.01000e+00</td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>13.75       </td><td>  7.41      </td><td>small       </td><td>negligible  </td><td> 7.01200e+00</td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>14.85       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>-8.32586e+17</td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 18.97          & -27.74         & negligible     & negligible     &  1.59100e+01   & FALSE          & TRUE          \\\\\n",
       "\t ericsson     & 14.86        &  -0.07       & small        & negligible   &  8.01000e+00 &  TRUE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 13.75          &   7.41         & small          & negligible     &  7.01200e+00   &  TRUE          & TRUE          \\\\\n",
       "\t naive        & 14.85        &   0.00       & negligible   & negligible   & -8.32586e+17 &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 18.97        | -27.74       | negligible   | negligible   |  1.59100e+01 | FALSE        | TRUE         | \n",
       "| ericsson     | 14.86        |  -0.07       | small        | negligible   |  8.01000e+00 |  TRUE        | TRUE         | \n",
       "| ets_th:all   | 13.75        |   7.41       | small        | negligible   |  7.01200e+00 |  TRUE        | TRUE         | \n",
       "| naive        | 14.85        |   0.00       | negligible   | negligible   | -8.32586e+17 |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          Wilcoxon_test\n",
       "1 arima_th:all 18.97   -27.74 negligible  negligible  1.59100e+01 FALSE        \n",
       "2 ericsson     14.86    -0.07 small       negligible  8.01000e+00  TRUE        \n",
       "3 ets_th:all   13.75     7.41 small       negligible  7.01200e+00  TRUE        \n",
       "4 naive        14.85     0.00 negligible  negligible -8.32586e+17    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'firefox'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'firefox'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'firefox'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\" \"firefox\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>60.17       </td><td>14.19       </td><td>negligible  </td><td>negligible  </td><td>1.247200e+01</td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>71.03       </td><td>-1.30       </td><td>small       </td><td>negligible  </td><td>6.164000e+00</td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>58.61       </td><td>16.41       </td><td>negligible  </td><td>negligible  </td><td>1.083200e+01</td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>70.12       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>3.850646e+17</td><td>  NA        </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 60.17          & 14.19          & negligible     & negligible     & 1.247200e+01   & TRUE           & FALSE         \\\\\n",
       "\t ericsson     & 71.03        & -1.30        & small        & negligible   & 6.164000e+00 & TRUE         & FALSE       \\\\\n",
       "\t ets\\_th:all   & 58.61          & 16.41          & negligible     & negligible     & 1.083200e+01   & TRUE           & FALSE         \\\\\n",
       "\t naive        & 70.12        &  0.00        & negligible   & negligible   & 3.850646e+17 &   NA         & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 60.17        | 14.19        | negligible   | negligible   | 1.247200e+01 | TRUE         | FALSE        | \n",
       "| ericsson     | 71.03        | -1.30        | small        | negligible   | 6.164000e+00 | TRUE         | FALSE        | \n",
       "| ets_th:all   | 58.61        | 16.41        | negligible   | negligible   | 1.083200e+01 | TRUE         | FALSE        | \n",
       "| naive        | 70.12        |  0.00        | negligible   | negligible   | 3.850646e+17 |   NA         | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          Wilcoxon_test\n",
       "1 arima_th:all 60.17   14.19  negligible  negligible 1.247200e+01 TRUE         \n",
       "2 ericsson     71.03   -1.30  small       negligible 6.164000e+00 TRUE         \n",
       "3 ets_th:all   58.61   16.41  negligible  negligible 1.083200e+01 TRUE         \n",
       "4 naive        70.12    0.00  negligible  negligible 3.850646e+17   NA         \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'core'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'core'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'core'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\" \"core\"   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>117.35       </td><td>18.77        </td><td>small        </td><td>negligible   </td><td> 4.221000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>131.98       </td><td> 8.65        </td><td>small        </td><td>negligible   </td><td> 3.978000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>125.02       </td><td>13.46        </td><td>small        </td><td>negligible   </td><td> 3.997000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>144.47       </td><td> 0.00        </td><td>negligible   </td><td>negligible   </td><td>-9.363283e+17</td><td>  NA         </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 117.35         & 18.77          & small          & negligible     &  4.221000e+00  & TRUE           & TRUE          \\\\\n",
       "\t ericsson      & 131.98        &  8.65         & small         & negligible    &  3.978000e+00 & TRUE          & TRUE         \\\\\n",
       "\t ets\\_th:all   & 125.02         & 13.46          & small          & negligible     &  3.997000e+00  & TRUE           & TRUE          \\\\\n",
       "\t naive         & 144.47        &  0.00         & negligible    & negligible    & -9.363283e+17 &   NA          & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 117.35        | 18.77         | small         | negligible    |  4.221000e+00 | TRUE          | TRUE          | \n",
       "| ericsson      | 131.98        |  8.65         | small         | negligible    |  3.978000e+00 | TRUE          | TRUE          | \n",
       "| ets_th:all    | 125.02        | 13.46         | small         | negligible    |  3.997000e+00 | TRUE          | TRUE          | \n",
       "| naive         | 144.47        |  0.00         | negligible    | negligible    | -9.363283e+17 |   NA          | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 117.35  18.77  small       negligible  4.221000e+00\n",
       "2 ericsson     131.98   8.65  small       negligible  3.978000e+00\n",
       "3 ets_th:all   125.02  13.46  small       negligible  3.997000e+00\n",
       "4 naive        144.47   0.00  negligible  negligible -9.363283e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 TRUE          TRUE              \n",
       "2 TRUE          TRUE              \n",
       "3 TRUE          TRUE              \n",
       "4   NA          TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'mozilla'</li>\n",
       "\t<li>'thunderbird'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'mozilla'\n",
       "\\item 'thunderbird'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'mozilla'\n",
       "2. 'thunderbird'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"mozilla\"     \"thunderbird\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>18.70       </td><td>15.15       </td><td>negligible  </td><td>negligible  </td><td>9.08300e+00 </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>20.94       </td><td> 4.99       </td><td>negligible  </td><td>negligible  </td><td>1.00820e+01 </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>19.52       </td><td>11.43       </td><td>small       </td><td>negligible  </td><td>8.33300e+00 </td><td>TRUE        </td><td>FALSE       </td></tr>\n",
       "\t<tr><td>naive       </td><td>22.04       </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>3.42837e+17 </td><td>  NA        </td><td>FALSE       </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 18.70          & 15.15          & negligible     & negligible     & 9.08300e+00    & TRUE           & FALSE         \\\\\n",
       "\t ericsson     & 20.94        &  4.99        & negligible   & negligible   & 1.00820e+01  & TRUE         & FALSE       \\\\\n",
       "\t ets\\_th:all   & 19.52          & 11.43          & small          & negligible     & 8.33300e+00    & TRUE           & FALSE         \\\\\n",
       "\t naive        & 22.04        &  0.00        & negligible   & negligible   & 3.42837e+17  &   NA         & FALSE       \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 18.70        | 15.15        | negligible   | negligible   | 9.08300e+00  | TRUE         | FALSE        | \n",
       "| ericsson     | 20.94        |  4.99        | negligible   | negligible   | 1.00820e+01  | TRUE         | FALSE        | \n",
       "| ets_th:all   | 19.52        | 11.43        | small        | negligible   | 8.33300e+00  | TRUE         | FALSE        | \n",
       "| naive        | 22.04        |  0.00        | negligible   | negligible   | 3.42837e+17  |   NA         | FALSE        | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT         Wilcoxon_test\n",
       "1 arima_th:all 18.70   15.15  negligible  negligible 9.08300e+00 TRUE         \n",
       "2 ericsson     20.94    4.99  negligible  negligible 1.00820e+01 TRUE         \n",
       "3 ets_th:all   19.52   11.43  small       negligible 8.33300e+00 TRUE         \n",
       "4 naive        22.04    0.00  negligible  negligible 3.42837e+17   NA         \n",
       "  Independent_errors\n",
       "1 FALSE             \n",
       "2 FALSE             \n",
       "3 FALSE             \n",
       "4 FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'kernel'</li>\n",
       "\t<li>'IO_Storage'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'kernel'\n",
       "\\item 'IO\\_Storage'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'kernel'\n",
       "2. 'IO_Storage'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"kernel\"     \"IO_Storage\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>2.80         </td><td> -2.94       </td><td>negligible   </td><td>negligible   </td><td>-2.204100e+01</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>3.19         </td><td>-17.28       </td><td>negligible   </td><td>negligible   </td><td>-1.087600e+01</td><td> TRUE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>2.77         </td><td> -1.84       </td><td>negligible   </td><td>negligible   </td><td>-2.730900e+01</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>2.72         </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td>-7.423934e+16</td><td>   NA        </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.80           &  -2.94         & negligible     & negligible     & -2.204100e+01  & FALSE          & TRUE          \\\\\n",
       "\t ericsson      & 3.19          & -17.28        & negligible    & negligible    & -1.087600e+01 &  TRUE         & TRUE         \\\\\n",
       "\t ets\\_th:all   & 2.77           &  -1.84         & negligible     & negligible     & -2.730900e+01  & FALSE          & TRUE          \\\\\n",
       "\t naive         & 2.72          &   0.00        & negligible    & negligible    & -7.423934e+16 &    NA         & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 2.80          |  -2.94        | negligible    | negligible    | -2.204100e+01 | FALSE         | TRUE          | \n",
       "| ericsson      | 3.19          | -17.28        | negligible    | negligible    | -1.087600e+01 |  TRUE         | TRUE          | \n",
       "| ets_th:all    | 2.77          |  -1.84        | negligible    | negligible    | -2.730900e+01 | FALSE         | TRUE          | \n",
       "| naive         | 2.72          |   0.00        | negligible    | negligible    | -7.423934e+16 |    NA         | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 2.80     -2.94 negligible  negligible -2.204100e+01\n",
       "2 ericsson     3.19    -17.28 negligible  negligible -1.087600e+01\n",
       "3 ets_th:all   2.77     -1.84 negligible  negligible -2.730900e+01\n",
       "4 naive        2.72      0.00 negligible  negligible -7.423934e+16\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         TRUE              \n",
       "2  TRUE         TRUE              \n",
       "3 FALSE         TRUE              \n",
       "4    NA         TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'kernel'</li>\n",
       "\t<li>'file_system'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'kernel'\n",
       "\\item 'file\\_system'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'kernel'\n",
       "2. 'file_system'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"kernel\"      \"file_system\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>2.68         </td><td>  5.30       </td><td>negligible   </td><td>negligible   </td><td> 4.102300e+01</td><td> TRUE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>3.50         </td><td>-23.67       </td><td>negligible   </td><td>negligible   </td><td>-2.908600e+01</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>2.74         </td><td>  3.18       </td><td>negligible   </td><td>negligible   </td><td> 1.626590e+02</td><td>FALSE        </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>2.83         </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td>-4.593834e+17</td><td>   NA        </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 2.68           &   5.30         & negligible     & negligible     &  4.102300e+01  &  TRUE          & TRUE          \\\\\n",
       "\t ericsson      & 3.50          & -23.67        & negligible    & negligible    & -2.908600e+01 & FALSE         & TRUE         \\\\\n",
       "\t ets\\_th:all   & 2.74           &   3.18         & negligible     & negligible     &  1.626590e+02  & FALSE          & TRUE          \\\\\n",
       "\t naive         & 2.83          &   0.00        & negligible    & negligible    & -4.593834e+17 &    NA         & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 2.68          |   5.30        | negligible    | negligible    |  4.102300e+01 |  TRUE         | TRUE          | \n",
       "| ericsson      | 3.50          | -23.67        | negligible    | negligible    | -2.908600e+01 | FALSE         | TRUE          | \n",
       "| ets_th:all    | 2.74          |   3.18        | negligible    | negligible    |  1.626590e+02 | FALSE         | TRUE          | \n",
       "| naive         | 2.83          |   0.00        | negligible    | negligible    | -4.593834e+17 |    NA         | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 2.68      5.30 negligible  negligible  4.102300e+01\n",
       "2 ericsson     3.50    -23.67 negligible  negligible -2.908600e+01\n",
       "3 ets_th:all   2.74      3.18 negligible  negligible  1.626590e+02\n",
       "4 naive        2.83      0.00 negligible  negligible -4.593834e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1  TRUE         TRUE              \n",
       "2 FALSE         TRUE              \n",
       "3 FALSE         TRUE              \n",
       "4    NA         TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'open_office'</li>\n",
       "\t<li>'writer'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'open\\_office'\n",
       "\\item 'writer'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'open_office'\n",
       "2. 'writer'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"open_office\" \"writer\"     "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>46.25        </td><td>  0.02       </td><td>negligible   </td><td>negligible   </td><td> 1.006700e+01</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>54.39        </td><td>-17.57       </td><td>small        </td><td>negligible   </td><td> 3.953000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>44.70        </td><td>  3.37       </td><td>negligible   </td><td>negligible   </td><td> 7.135700e+01</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>46.26        </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td>-1.368751e+18</td><td>  NA         </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 46.25          &   0.02         & negligible     & negligible     &  1.006700e+01  & TRUE           & TRUE          \\\\\n",
       "\t ericsson      & 54.39         & -17.57        & small         & negligible    &  3.953000e+00 & TRUE          & TRUE         \\\\\n",
       "\t ets\\_th:all   & 44.70          &   3.37         & negligible     & negligible     &  7.135700e+01  & TRUE           & TRUE          \\\\\n",
       "\t naive         & 46.26         &   0.00        & negligible    & negligible    & -1.368751e+18 &   NA          & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 46.25         |   0.02        | negligible    | negligible    |  1.006700e+01 | TRUE          | TRUE          | \n",
       "| ericsson      | 54.39         | -17.57        | small         | negligible    |  3.953000e+00 | TRUE          | TRUE          | \n",
       "| ets_th:all    | 44.70         |   3.37        | negligible    | negligible    |  7.135700e+01 | TRUE          | TRUE          | \n",
       "| naive         | 46.26         |   0.00        | negligible    | negligible    | -1.368751e+18 |   NA          | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 46.25     0.02 negligible  negligible  1.006700e+01\n",
       "2 ericsson     54.39   -17.57 small       negligible  3.953000e+00\n",
       "3 ets_th:all   44.70     3.37 negligible  negligible  7.135700e+01\n",
       "4 naive        46.26     0.00 negligible  negligible -1.368751e+18\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 TRUE          TRUE              \n",
       "2 TRUE          TRUE              \n",
       "3 TRUE          TRUE              \n",
       "4   NA          TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'open_office'</li>\n",
       "\t<li>'calc'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'open\\_office'\n",
       "\\item 'calc'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'open_office'\n",
       "2. 'calc'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"open_office\" \"calc\"       "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>17.21        </td><td>-24.35       </td><td>small        </td><td>negligible   </td><td> 8.224000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>19.18        </td><td>-38.58       </td><td>small        </td><td>negligible   </td><td> 3.885000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>11.70        </td><td> 15.46       </td><td>small        </td><td>negligible   </td><td> 5.408000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>13.84        </td><td>  0.00       </td><td>negligible   </td><td>negligible   </td><td>-1.938129e+17</td><td>  NA         </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 17.21          & -24.35         & small          & negligible     &  8.224000e+00  & TRUE           & TRUE          \\\\\n",
       "\t ericsson      & 19.18         & -38.58        & small         & negligible    &  3.885000e+00 & TRUE          & TRUE         \\\\\n",
       "\t ets\\_th:all   & 11.70          &  15.46         & small          & negligible     &  5.408000e+00  & TRUE           & TRUE          \\\\\n",
       "\t naive         & 13.84         &   0.00        & negligible    & negligible    & -1.938129e+17 &   NA          & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 17.21         | -24.35        | small         | negligible    |  8.224000e+00 | TRUE          | TRUE          | \n",
       "| ericsson      | 19.18         | -38.58        | small         | negligible    |  3.885000e+00 | TRUE          | TRUE          | \n",
       "| ets_th:all    | 11.70         |  15.46        | small         | negligible    |  5.408000e+00 | TRUE          | TRUE          | \n",
       "| naive         | 13.84         |   0.00        | negligible    | negligible    | -1.938129e+17 |   NA          | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 17.21   -24.35 small       negligible  8.224000e+00\n",
       "2 ericsson     19.18   -38.58 small       negligible  3.885000e+00\n",
       "3 ets_th:all   11.70    15.46 small       negligible  5.408000e+00\n",
       "4 naive        13.84     0.00 negligible  negligible -1.938129e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 TRUE          TRUE              \n",
       "2 TRUE          TRUE              \n",
       "3 TRUE          TRUE              \n",
       "4   NA          TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'open_office'</li>\n",
       "\t<li>'draw'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'open\\_office'\n",
       "\\item 'draw'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'open_office'\n",
       "2. 'draw'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"open_office\" \"draw\"       "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>6.24         </td><td>-130.26      </td><td>negligible   </td><td>negligible   </td><td>-1.079600e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>4.96         </td><td> -83.03      </td><td>negligible   </td><td>negligible   </td><td>-3.908200e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>3.84         </td><td> -41.70      </td><td>negligible   </td><td>negligible   </td><td>-2.595500e+01</td><td>FALSE        </td><td>FALSE        </td></tr>\n",
       "\t<tr><td>naive        </td><td>2.71         </td><td>   0.00      </td><td>negligible   </td><td>negligible   </td><td> 2.097314e+17</td><td>   NA        </td><td>FALSE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 6.24           & -130.26        & negligible     & negligible     & -1.079600e+01  & FALSE          & FALSE         \\\\\n",
       "\t ericsson      & 4.96          &  -83.03       & negligible    & negligible    & -3.908200e+01 & FALSE         & FALSE        \\\\\n",
       "\t ets\\_th:all   & 3.84           &  -41.70        & negligible     & negligible     & -2.595500e+01  & FALSE          & FALSE         \\\\\n",
       "\t naive         & 2.71          &    0.00       & negligible    & negligible    &  2.097314e+17 &    NA         & FALSE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 6.24          | -130.26       | negligible    | negligible    | -1.079600e+01 | FALSE         | FALSE         | \n",
       "| ericsson      | 4.96          |  -83.03       | negligible    | negligible    | -3.908200e+01 | FALSE         | FALSE         | \n",
       "| ets_th:all    | 3.84          |  -41.70       | negligible    | negligible    | -2.595500e+01 | FALSE         | FALSE         | \n",
       "| naive         | 2.71          |    0.00       | negligible    | negligible    |  2.097314e+17 |    NA         | FALSE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1  Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 6.24    -130.26 negligible  negligible -1.079600e+01\n",
       "2 ericsson     4.96     -83.03 negligible  negligible -3.908200e+01\n",
       "3 ets_th:all   3.84     -41.70 negligible  negligible -2.595500e+01\n",
       "4 naive        2.71       0.00 negligible  negligible  2.097314e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 FALSE         FALSE             \n",
       "2 FALSE         FALSE             \n",
       "3 FALSE         FALSE             \n",
       "4    NA         FALSE             "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'apache'</li>\n",
       "\t<li>'ant'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'apache'\n",
       "\\item 'ant'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'apache'\n",
       "2. 'ant'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"apache\" \"ant\"   "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>20.12        </td><td>11.33        </td><td>small        </td><td>negligible   </td><td> 5.854000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>19.79        </td><td>12.78        </td><td>small        </td><td>negligible   </td><td> 8.048000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>20.43        </td><td> 9.96        </td><td>negligible   </td><td>negligible   </td><td> 1.541200e+01</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>22.69        </td><td> 0.00        </td><td>negligible   </td><td>negligible   </td><td>-1.859277e+17</td><td>  NA         </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 20.12          & 11.33          & small          & negligible     &  5.854000e+00  & TRUE           & TRUE          \\\\\n",
       "\t ericsson      & 19.79         & 12.78         & small         & negligible    &  8.048000e+00 & TRUE          & TRUE         \\\\\n",
       "\t ets\\_th:all   & 20.43          &  9.96          & negligible     & negligible     &  1.541200e+01  & TRUE           & TRUE          \\\\\n",
       "\t naive         & 22.69         &  0.00         & negligible    & negligible    & -1.859277e+17 &   NA          & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 20.12         | 11.33         | small         | negligible    |  5.854000e+00 | TRUE          | TRUE          | \n",
       "| ericsson      | 19.79         | 12.78         | small         | negligible    |  8.048000e+00 | TRUE          | TRUE          | \n",
       "| ets_th:all    | 20.43         |  9.96         | negligible    | negligible    |  1.541200e+01 | TRUE          | TRUE          | \n",
       "| naive         | 22.69         |  0.00         | negligible    | negligible    | -1.859277e+17 |   NA          | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 20.12   11.33  small       negligible  5.854000e+00\n",
       "2 ericsson     19.79   12.78  small       negligible  8.048000e+00\n",
       "3 ets_th:all   20.43    9.96  negligible  negligible  1.541200e+01\n",
       "4 naive        22.69    0.00  negligible  negligible -1.859277e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 TRUE          TRUE              \n",
       "2 TRUE          TRUE              \n",
       "3 TRUE          TRUE              \n",
       "4   NA          TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'apache'</li>\n",
       "\t<li>'apache2'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'apache'\n",
       "\\item 'apache2'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'apache'\n",
       "2. 'apache2'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"apache\"  \"apache2\""
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td> 6.13       </td><td> 20.49      </td><td>negligible  </td><td>negligible  </td><td>1.172600e+01</td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>10.49       </td><td>-36.06      </td><td>negligible  </td><td>negligible  </td><td>1.308170e+02</td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td> 7.36       </td><td>  4.54      </td><td>negligible  </td><td>negligible  </td><td>1.386900e+01</td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td> 7.71       </td><td>  0.00      </td><td>negligible  </td><td>negligible  </td><td>1.612504e+17</td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all &  6.13          &  20.49         & negligible     & negligible     & 1.172600e+01   &  TRUE          & TRUE          \\\\\n",
       "\t ericsson     & 10.49        & -36.06       & negligible   & negligible   & 1.308170e+02 & FALSE        & TRUE        \\\\\n",
       "\t ets\\_th:all   &  7.36          &   4.54         & negligible     & negligible     & 1.386900e+01   &  TRUE          & TRUE          \\\\\n",
       "\t naive        &  7.71        &   0.00       & negligible   & negligible   & 1.612504e+17 &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all |  6.13        |  20.49       | negligible   | negligible   | 1.172600e+01 |  TRUE        | TRUE         | \n",
       "| ericsson     | 10.49        | -36.06       | negligible   | negligible   | 1.308170e+02 | FALSE        | TRUE         | \n",
       "| ets_th:all   |  7.36        |   4.54       | negligible   | negligible   | 1.386900e+01 |  TRUE        | TRUE         | \n",
       "| naive        |  7.71        |   0.00       | negligible   | negligible   | 1.612504e+17 |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          Wilcoxon_test\n",
       "1 arima_th:all  6.13    20.49 negligible  negligible 1.172600e+01  TRUE        \n",
       "2 ericsson     10.49   -36.06 negligible  negligible 1.308170e+02 FALSE        \n",
       "3 ets_th:all    7.36     4.54 negligible  negligible 1.386900e+01  TRUE        \n",
       "4 naive         7.71     0.00 negligible  negligible 1.612504e+17    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'libre_office'</li>\n",
       "\t<li>'writer'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'libre\\_office'\n",
       "\\item 'writer'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'libre_office'\n",
       "2. 'writer'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"libre_office\" \"writer\"      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>20.53        </td><td>28.69        </td><td>medium       </td><td>small        </td><td> 3.188000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>24.19        </td><td>15.98        </td><td>small        </td><td>negligible   </td><td> 4.020000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>20.90        </td><td>27.41        </td><td>medium       </td><td>small        </td><td> 3.410000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>28.79        </td><td> 0.00        </td><td>negligible   </td><td>negligible   </td><td>-3.952874e+18</td><td>  NA         </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 20.53          & 28.69          & medium         & small          &  3.188000e+00  & TRUE           & TRUE          \\\\\n",
       "\t ericsson      & 24.19         & 15.98         & small         & negligible    &  4.020000e+00 & TRUE          & TRUE         \\\\\n",
       "\t ets\\_th:all   & 20.90          & 27.41          & medium         & small          &  3.410000e+00  & TRUE           & TRUE          \\\\\n",
       "\t naive         & 28.79         &  0.00         & negligible    & negligible    & -3.952874e+18 &   NA          & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 20.53         | 28.69         | medium        | small         |  3.188000e+00 | TRUE          | TRUE          | \n",
       "| ericsson      | 24.19         | 15.98         | small         | negligible    |  4.020000e+00 | TRUE          | TRUE          | \n",
       "| ets_th:all    | 20.90         | 27.41         | medium        | small         |  3.410000e+00 | TRUE          | TRUE          | \n",
       "| naive         | 28.79         |  0.00         | negligible    | negligible    | -3.952874e+18 |   NA          | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 20.53   28.69  medium      small       3.188000e+00\n",
       "2 ericsson     24.19   15.98  small       negligible  4.020000e+00\n",
       "3 ets_th:all   20.90   27.41  medium      small       3.410000e+00\n",
       "4 naive        28.79    0.00  negligible  negligible -3.952874e+18\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 TRUE          TRUE              \n",
       "2 TRUE          TRUE              \n",
       "3 TRUE          TRUE              \n",
       "4   NA          TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'libre_office'</li>\n",
       "\t<li>'calc'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'libre\\_office'\n",
       "\\item 'calc'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'libre_office'\n",
       "2. 'calc'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"libre_office\" \"calc\"        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all </td><td>11.38        </td><td>35.41        </td><td>medium       </td><td>small        </td><td> 2.614000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ericsson     </td><td>12.69        </td><td>27.98        </td><td>medium       </td><td>small        </td><td> 2.992000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>ets_th:all   </td><td>12.01        </td><td>31.84        </td><td>medium       </td><td>small        </td><td> 2.942000e+00</td><td>TRUE         </td><td>TRUE         </td></tr>\n",
       "\t<tr><td>naive        </td><td>17.62        </td><td> 0.00        </td><td>negligible   </td><td>negligible   </td><td>-2.269734e+17</td><td>  NA         </td><td>TRUE         </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 11.38          & 35.41          & medium         & small          &  2.614000e+00  & TRUE           & TRUE          \\\\\n",
       "\t ericsson      & 12.69         & 27.98         & medium        & small         &  2.992000e+00 & TRUE          & TRUE         \\\\\n",
       "\t ets\\_th:all   & 12.01          & 31.84          & medium         & small          &  2.942000e+00  & TRUE           & TRUE          \\\\\n",
       "\t naive         & 17.62         &  0.00         & negligible    & negligible    & -2.269734e+17 &   NA          & TRUE         \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all  | 11.38         | 35.41         | medium        | small         |  2.614000e+00 | TRUE          | TRUE          | \n",
       "| ericsson      | 12.69         | 27.98         | medium        | small         |  2.992000e+00 | TRUE          | TRUE          | \n",
       "| ets_th:all    | 12.01         | 31.84         | medium        | small         |  2.942000e+00 | TRUE          | TRUE          | \n",
       "| naive         | 17.62         |  0.00         | negligible    | negligible    | -2.269734e+17 |   NA          | TRUE          | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT          \n",
       "1 arima_th:all 11.38   35.41  medium      small       2.614000e+00\n",
       "2 ericsson     12.69   27.98  medium      small       2.992000e+00\n",
       "3 ets_th:all   12.01   31.84  medium      small       2.942000e+00\n",
       "4 naive        17.62    0.00  negligible  negligible -2.269734e+17\n",
       "  Wilcoxon_test Independent_errors\n",
       "1 TRUE          TRUE              \n",
       "2 TRUE          TRUE              \n",
       "3 TRUE          TRUE              \n",
       "4   NA          TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'libre_office'</li>\n",
       "\t<li>'draw'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'libre\\_office'\n",
       "\\item 'draw'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'libre_office'\n",
       "2. 'draw'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"libre_office\" \"draw\"        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Method</th><th scope=col>Error_1</th><th scope=col>Impr_1</th><th scope=col>Cliff_delta</th><th scope=col>Hedges_g</th><th scope=col>NNT</th><th scope=col>Wilcoxon_test</th><th scope=col>Independent_errors</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>arima_th:all</td><td>3.46        </td><td>11.51       </td><td>negligible  </td><td>negligible  </td><td>1.44420e+01 </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ericsson    </td><td>3.43        </td><td>12.28       </td><td>negligible  </td><td>negligible  </td><td>9.51600e+00 </td><td>FALSE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>ets_th:all  </td><td>3.51        </td><td>10.23       </td><td>negligible  </td><td>negligible  </td><td>1.48440e+01 </td><td> TRUE       </td><td>TRUE        </td></tr>\n",
       "\t<tr><td>naive       </td><td>3.91        </td><td> 0.00       </td><td>negligible  </td><td>negligible  </td><td>1.42345e+18 </td><td>   NA       </td><td>TRUE        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llllllll}\n",
       " Method & Error\\_1 & Impr\\_1 & Cliff\\_delta & Hedges\\_g & NNT & Wilcoxon\\_test & Independent\\_errors\\\\\n",
       "\\hline\n",
       "\t arima\\_th:all & 3.46           & 11.51          & negligible     & negligible     & 1.44420e+01    &  TRUE          & TRUE          \\\\\n",
       "\t ericsson     & 3.43         & 12.28        & negligible   & negligible   & 9.51600e+00  & FALSE        & TRUE        \\\\\n",
       "\t ets\\_th:all   & 3.51           & 10.23          & negligible     & negligible     & 1.48440e+01    &  TRUE          & TRUE          \\\\\n",
       "\t naive        & 3.91         &  0.00        & negligible   & negligible   & 1.42345e+18  &    NA        & TRUE        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Method | Error_1 | Impr_1 | Cliff_delta | Hedges_g | NNT | Wilcoxon_test | Independent_errors | \n",
       "|---|---|---|---|\n",
       "| arima_th:all | 3.46         | 11.51        | negligible   | negligible   | 1.44420e+01  |  TRUE        | TRUE         | \n",
       "| ericsson     | 3.43         | 12.28        | negligible   | negligible   | 9.51600e+00  | FALSE        | TRUE         | \n",
       "| ets_th:all   | 3.51         | 10.23        | negligible   | negligible   | 1.48440e+01  |  TRUE        | TRUE         | \n",
       "| naive        | 3.91         |  0.00        | negligible   | negligible   | 1.42345e+18  |    NA        | TRUE         | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "  Method       Error_1 Impr_1 Cliff_delta Hedges_g   NNT         Wilcoxon_test\n",
       "1 arima_th:all 3.46    11.51  negligible  negligible 1.44420e+01  TRUE        \n",
       "2 ericsson     3.43    12.28  negligible  negligible 9.51600e+00 FALSE        \n",
       "3 ets_th:all   3.51    10.23  negligible  negligible 1.48440e+01  TRUE        \n",
       "4 naive        3.91     0.00  negligible  negligible 1.42345e+18    NA        \n",
       "  Independent_errors\n",
       "1 TRUE              \n",
       "2 TRUE              \n",
       "3 TRUE              \n",
       "4 TRUE              "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Products</th><th scope=col>Arima</th><th scope=col>Ets</th><th scope=col>Ericsson</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>kernel_networking  </td><td> 0                 </td><td>0                  </td><td>-1                 </td></tr>\n",
       "\t<tr><td>eclipse_platform   </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>eclipse_data_tools </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>eclipse_birt       </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>eclipse_jdt        </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>eclipse_pde        </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>mozilla_calendar   </td><td> 0                 </td><td>1                  </td><td>-1                 </td></tr>\n",
       "\t<tr><td>mozilla_firefox    </td><td> 0                 </td><td>0                  </td><td>-1                 </td></tr>\n",
       "\t<tr><td>mozilla_core       </td><td> 1                 </td><td>1                  </td><td> 1                 </td></tr>\n",
       "\t<tr><td>mozilla_thunderbird</td><td> 0                 </td><td>1                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>kernel_IO_Storage  </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>kernel_file_system </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>open_office_writer </td><td> 0                 </td><td>0                  </td><td>-1                 </td></tr>\n",
       "\t<tr><td>open_office_calc   </td><td>-1                 </td><td>1                  </td><td>-1                 </td></tr>\n",
       "\t<tr><td>open_office_draw   </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>apache_ant         </td><td> 1                 </td><td>0                  </td><td> 1                 </td></tr>\n",
       "\t<tr><td>apache_apache2     </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "\t<tr><td>libre_office_writer</td><td> 1                 </td><td>1                  </td><td> 1                 </td></tr>\n",
       "\t<tr><td>libre_office_calc  </td><td> 1                 </td><td>1                  </td><td> 1                 </td></tr>\n",
       "\t<tr><td>libre_office_draw  </td><td> 0                 </td><td>0                  </td><td> 0                 </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|llll}\n",
       " Products & Arima & Ets & Ericsson\\\\\n",
       "\\hline\n",
       "\t kernel\\_networking   &  0                    & 0                     & -1                   \\\\\n",
       "\t eclipse\\_platform    &  0                    & 0                     &  0                   \\\\\n",
       "\t eclipse\\_data\\_tools  &  0                      & 0                       &  0                     \\\\\n",
       "\t eclipse\\_birt        &  0                    & 0                     &  0                   \\\\\n",
       "\t eclipse\\_jdt         &  0                    & 0                     &  0                   \\\\\n",
       "\t eclipse\\_pde         &  0                    & 0                     &  0                   \\\\\n",
       "\t mozilla\\_calendar    &  0                    & 1                     & -1                   \\\\\n",
       "\t mozilla\\_firefox     &  0                    & 0                     & -1                   \\\\\n",
       "\t mozilla\\_core        &  1                    & 1                     &  1                   \\\\\n",
       "\t mozilla\\_thunderbird &  0                    & 1                     &  0                   \\\\\n",
       "\t kernel\\_IO\\_Storage   &  0                      & 0                       &  0                     \\\\\n",
       "\t kernel\\_file\\_system  &  0                      & 0                       &  0                     \\\\\n",
       "\t open\\_office\\_writer  &  0                      & 0                       & -1                     \\\\\n",
       "\t open\\_office\\_calc    & -1                      & 1                       & -1                     \\\\\n",
       "\t open\\_office\\_draw    &  0                      & 0                       &  0                     \\\\\n",
       "\t apache\\_ant          &  1                    & 0                     &  1                   \\\\\n",
       "\t apache\\_apache2      &  0                    & 0                     &  0                   \\\\\n",
       "\t libre\\_office\\_writer &  1                      & 1                       &  1                     \\\\\n",
       "\t libre\\_office\\_calc   &  1                      & 1                       &  1                     \\\\\n",
       "\t libre\\_office\\_draw   &  0                      & 0                       &  0                     \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "Products | Arima | Ets | Ericsson | \n",
       "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n",
       "| kernel_networking   |  0                  | 0                   | -1                  | \n",
       "| eclipse_platform    |  0                  | 0                   |  0                  | \n",
       "| eclipse_data_tools  |  0                  | 0                   |  0                  | \n",
       "| eclipse_birt        |  0                  | 0                   |  0                  | \n",
       "| eclipse_jdt         |  0                  | 0                   |  0                  | \n",
       "| eclipse_pde         |  0                  | 0                   |  0                  | \n",
       "| mozilla_calendar    |  0                  | 1                   | -1                  | \n",
       "| mozilla_firefox     |  0                  | 0                   | -1                  | \n",
       "| mozilla_core        |  1                  | 1                   |  1                  | \n",
       "| mozilla_thunderbird |  0                  | 1                   |  0                  | \n",
       "| kernel_IO_Storage   |  0                  | 0                   |  0                  | \n",
       "| kernel_file_system  |  0                  | 0                   |  0                  | \n",
       "| open_office_writer  |  0                  | 0                   | -1                  | \n",
       "| open_office_calc    | -1                  | 1                   | -1                  | \n",
       "| open_office_draw    |  0                  | 0                   |  0                  | \n",
       "| apache_ant          |  1                  | 0                   |  1                  | \n",
       "| apache_apache2      |  0                  | 0                   |  0                  | \n",
       "| libre_office_writer |  1                  | 1                   |  1                  | \n",
       "| libre_office_calc   |  1                  | 1                   |  1                  | \n",
       "| libre_office_draw   |  0                  | 0                   |  0                  | \n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "   Products            Arima Ets Ericsson\n",
       "1  kernel_networking    0    0   -1      \n",
       "2  eclipse_platform     0    0    0      \n",
       "3  eclipse_data_tools   0    0    0      \n",
       "4  eclipse_birt         0    0    0      \n",
       "5  eclipse_jdt          0    0    0      \n",
       "6  eclipse_pde          0    0    0      \n",
       "7  mozilla_calendar     0    1   -1      \n",
       "8  mozilla_firefox      0    0   -1      \n",
       "9  mozilla_core         1    1    1      \n",
       "10 mozilla_thunderbird  0    1    0      \n",
       "11 kernel_IO_Storage    0    0    0      \n",
       "12 kernel_file_system   0    0    0      \n",
       "13 open_office_writer   0    0   -1      \n",
       "14 open_office_calc    -1    1   -1      \n",
       "15 open_office_draw     0    0    0      \n",
       "16 apache_ant           1    0    1      \n",
       "17 apache_apache2       0    0    0      \n",
       "18 libre_office_writer  1    1    1      \n",
       "19 libre_office_calc    1    1    1      \n",
       "20 libre_office_draw    0    0    0      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] \"Arima vs naive\"\n",
      "[1] \"4 : 1\"\n",
      "[1] \"ETS vs naive\"\n",
      "[1] \"6 : 0\"\n",
      "[1] \"Ericsson vs naive\"\n",
      "[1] \"4 : 5\"\n"
     ]
    }
   ],
   "source": [
    "compare_methods(products, \"naive\", 0)\n",
    "#compare_methods(products, \"naive\", 1)\n",
    "#compare_methods(products, \"ericsson\", 0)\n",
    "#compare_methods(products, \"ericsson\", 1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R 3.3",
   "language": "R",
   "name": "ir33"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}