SKAB_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import ruptures as rpt\n",
    "from utils.custom_cost import CostNew\n",
    "from SKAB_data.evaluating import evaluating_change_point"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Data importing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SKAB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# benchmark files checking\n",
    "all_files=[]\n",
    "import os\n",
    "for root, dirs, files in os.walk(\"SKAB_data/\"):\n",
    "    for file in files:\n",
    "        if file.endswith(\".csv\"):\n",
    "             all_files.append(os.path.join(root, file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# datasets with anomalies loading\n",
    "test = [pd.read_csv(file, \n",
    "                          sep=';', \n",
    "                          index_col='datetime', \n",
    "                          parse_dates=True).drop('anomaly', axis=1) for file in all_files if 'anomaly-free' not in file]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 Testing ensembling functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "signal = StandardScaler().fit_transform(test[1].drop('changepoint', axis=1).values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Testing Dynp and DynpEnsembling algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dynp algorithm:\n",
      "Average delay nan\n",
      "A number of missed CPs = 4\n",
      "Standart  -  -1.38\n",
      "LowFP  -  -2.75\n",
      "LowFN  -  -0.92\n"
     ]
    }
   ],
   "source": [
    "print('Dynp algorithm:')\n",
    "algo = rpt.Dynp(custom_cost='l1', jump=2).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=4)\n",
    "\n",
    "true_cp = test[1]['changepoint']\n",
    "predicted_cp = pd.Series(data=0, index=test[1].index)\n",
    "predicted_cp[predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "\n",
    "add = evaluating_change_point(true_cp, predicted_cp, metric='average_delay', numenta_time='30 sec')\n",
    "nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DynpEnsembling algorithm:\n",
      "Average delay 0 days 00:00:06\n",
      "A number of missed CPs = 3\n",
      "Standart  -  22.25\n",
      "LowFP  -  19.5\n",
      "LowFN  -  23.17\n"
     ]
    }
   ],
   "source": [
    "print('DynpEnsembling algorithm:')\n",
    "c = CostNew()\n",
    "algo = rpt.DynpEnsembling(custom_cost=c, jump=2, ensembling=5).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=4)\n",
    "\n",
    "true_cp = test[1]['changepoint']\n",
    "predicted_cp = pd.Series(data=0, index=test[1].index)\n",
    "predicted_cp[predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "\n",
    "add = evaluating_change_point(true_cp, predicted_cp, metric='average_delay', numenta_time='30 sec')\n",
    "nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Testing Window and WindowEnsembling algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Window algorithm:\n",
      "Average delay 0 days 00:00:05\n",
      "A number of missed CPs = 3\n",
      "Standart  -  25.0\n",
      "LowFP  -  25.0\n",
      "LowFN  -  25.0\n"
     ]
    }
   ],
   "source": [
    "print('Window algorithm:')\n",
    "algo = rpt.Window(custom_cost='ar', jump=1, width=40).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=4)\n",
    "\n",
    "true_cp = test[1]['changepoint']\n",
    "predicted_cp = pd.Series(data=0, index=test[1].index)\n",
    "predicted_cp[predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "\n",
    "add = evaluating_change_point(true_cp, predicted_cp, metric='average_delay', numenta_time='30 sec')\n",
    "nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WindowEnsembling algorithm:\n",
      "Average delay 0 days 00:00:05\n",
      "A number of missed CPs = 3\n",
      "Standart  -  25.0\n",
      "LowFP  -  25.0\n",
      "LowFN  -  25.0\n"
     ]
    }
   ],
   "source": [
    "print('WindowEnsembling algorithm:')\n",
    "c = CostNew()\n",
    "algo = rpt.WindowEnsembling(custom_cost=c, jump=1, width=40, ensembling=5).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=4)\n",
    "\n",
    "true_cp = test[1]['changepoint']\n",
    "predicted_cp = pd.Series(data=0, index=test[1].index)\n",
    "predicted_cp[predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "\n",
    "add = evaluating_change_point(true_cp, predicted_cp, metric='average_delay', numenta_time='30 sec')\n",
    "nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Testing BinSeg and BinSegEnsembling algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BinSeg algorithm:\n",
      "Average delay 0 days 00:00:15.500000\n",
      "A number of missed CPs = 2\n",
      "Standart  -  45.69\n",
      "LowFP  -  41.54\n",
      "LowFN  -  47.12\n"
     ]
    }
   ],
   "source": [
    "print('BinSeg algorithm:')\n",
    "algo = rpt.Binseg(custom_cost='l2', jump=1).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=4)\n",
    "\n",
    "true_cp = test[1]['changepoint']\n",
    "predicted_cp = pd.Series(data=0, index=test[1].index)\n",
    "predicted_cp[predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "\n",
    "add = evaluating_change_point(true_cp, predicted_cp, metric='average_delay', numenta_time='30 sec')\n",
    "nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BinSegEnsembling algorithm:\n",
      "Average delay 0 days 00:00:06\n",
      "A number of missed CPs = 3\n",
      "Standart  -  38.94\n",
      "LowFP  -  34.12\n",
      "LowFN  -  42.62\n"
     ]
    }
   ],
   "source": [
    "print('BinSegEnsembling algorithm:')\n",
    "c = CostNew()\n",
    "algo = rpt.BinsegEnsembling(custom_cost=c, jump=1, ensembling=5).fit(signal)\n",
    "\n",
    "my_bkps = algo.predict(n_bkps=4)\n",
    "\n",
    "true_cp = test[1]['changepoint']\n",
    "predicted_cp = pd.Series(data=0, index=test[1].index)\n",
    "predicted_cp[predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "\n",
    "add = evaluating_change_point(true_cp, predicted_cp, metric='average_delay', numenta_time='30 sec')\n",
    "nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CPDE = 17"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Dynp experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Individual algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dynp(cost, data, params):\n",
    "    predicted_cp = []\n",
    "    for dataset in data:\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(dataset.drop('changepoint', axis=1))\n",
    "        algo = rpt.Dynp(model=cost, \n",
    "                        params=params, \n",
    "                        jump=5)\n",
    "        algo.fit(signal)\n",
    "        my_bkps = algo.predict(n_bkps=len(dataset[dataset['changepoint']==1]))\n",
    "        \n",
    "        single_predicted_cp = pd.Series(data=0, index=dataset.index)\n",
    "        single_predicted_cp[single_predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "        predicted_cp.append(single_predicted_cp)\n",
    "\n",
    "    true_cp = [dataset.changepoint for dataset in data]\n",
    "    nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')\n",
    "    return nab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = (\n",
    "    {'cost':'ar', 'params':{'order':1}},\n",
    "    {'cost':'mahalanobis', 'params':{}},\n",
    "    {'cost':'l1', 'params':{}},\n",
    "    {'cost':'l2', 'params':{}},\n",
    "    {'cost':'linear', 'params':{}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7min 38s, sys: 13.2 s, total: 7min 51s\n",
      "Wall time: 7min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "table = []\n",
    "for model in tqdm_notebook(models):\n",
    "    results = dynp(data=test, **model)\n",
    "    clear_output()\n",
    "    table.append(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_55b245ac_8c1e_11eb_a67f_acde48001122row0_col0{\n",
       "            background-color:  #1c6bb0;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row0_col1{\n",
       "            background-color:  #1d6cb1;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row0_col2{\n",
       "            background-color:  #1967ad;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row1_col0,#T_55b245ac_8c1e_11eb_a67f_acde48001122row1_col1,#T_55b245ac_8c1e_11eb_a67f_acde48001122row1_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row2_col0{\n",
       "            background-color:  #2b7bba;\n",
       "            color:  #000000;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row2_col1{\n",
       "            background-color:  #2d7dbb;\n",
       "            color:  #000000;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row2_col2,#T_55b245ac_8c1e_11eb_a67f_acde48001122row3_col1{\n",
       "            background-color:  #2272b6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row3_col0{\n",
       "            background-color:  #2474b7;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row3_col2{\n",
       "            background-color:  #1e6db2;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_55b245ac_8c1e_11eb_a67f_acde48001122row4_col0,#T_55b245ac_8c1e_11eb_a67f_acde48001122row4_col1,#T_55b245ac_8c1e_11eb_a67f_acde48001122row4_col2{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Standart</th>        <th class=\"col_heading level0 col1\" >LowFP</th>        <th class=\"col_heading level0 col2\" >LowFN</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row0_col0\" class=\"data row0 col0\" >19.400000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row0_col1\" class=\"data row0 col1\" >16.830000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row0_col2\" class=\"data row0 col2\" >20.630000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row1_col0\" class=\"data row1 col0\" >22.370000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row1_col1\" class=\"data row1 col1\" >19.900000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row1_col2\" class=\"data row1 col2\" >23.370000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row2_col0\" class=\"data row2 col0\" >18.640000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row2_col1\" class=\"data row2 col1\" >15.990000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row2_col2\" class=\"data row2 col2\" >20.120000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row3_col0\" class=\"data row3 col0\" >18.960000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row3_col1\" class=\"data row3 col1\" >16.500000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row3_col2\" class=\"data row3 col2\" >20.330000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row4_col0\" class=\"data row4 col0\" >9.370000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row4_col1\" class=\"data row4 col1\" >6.600000</td>\n",
       "                        <td id=\"T_55b245ac_8c1e_11eb_a67f_acde48001122row4_col2\" class=\"data row4 col2\" >10.610000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1395eb5b0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(table).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dynpEnsemble(cost, data, num_agg_func):\n",
    "    predicted_cp = []\n",
    "    for dataset in data:\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(dataset.drop('changepoint', axis=1))\n",
    "        algo = rpt.DynpEnsembling(custom_cost=cost, \n",
    "                                  jump=5, \n",
    "                                  ensembling=num_agg_func)\n",
    "        single_predicted_cp = pd.Series(data=0, index=dataset.index)\n",
    "        try:\n",
    "            algo.fit(signal)\n",
    "            my_bkps = algo.predict(n_bkps=len(dataset[dataset['changepoint']==1]))\n",
    "\n",
    "            single_predicted_cp[single_predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "            predicted_cp.append(single_predicted_cp)\n",
    "        except:\n",
    "            predicted_cp.append(single_predicted_cp)\n",
    "    \n",
    "    true_cp = [dataset.changepoint for dataset in data]\n",
    "    nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')\n",
    "    return nab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2h 40min 18s, sys: 5min 20s, total: 2h 45min 39s\n",
      "Wall time: 2h 16min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cost = CostNew()\n",
    "table1 = []\n",
    "\n",
    "for n in tnrange(1, NUM_CPDE, desc='agg functions loop'):\n",
    "    table1.append(dynpEnsemble(cost=cost, data=test, num_agg_func=n))\n",
    "    clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_64592914_8c31_11eb_a67f_acde48001122row0_col0,#T_64592914_8c31_11eb_a67f_acde48001122row3_col0{\n",
       "            background-color:  #084e98;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row0_col1,#T_64592914_8c31_11eb_a67f_acde48001122row3_col1{\n",
       "            background-color:  #084d96;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row0_col2,#T_64592914_8c31_11eb_a67f_acde48001122row2_col1,#T_64592914_8c31_11eb_a67f_acde48001122row3_col2{\n",
       "            background-color:  #08509b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row1_col0,#T_64592914_8c31_11eb_a67f_acde48001122row1_col2{\n",
       "            background-color:  #1561a9;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row1_col1{\n",
       "            background-color:  #125da6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row2_col0{\n",
       "            background-color:  #09529d;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row2_col2{\n",
       "            background-color:  #08519c;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row4_col0,#T_64592914_8c31_11eb_a67f_acde48001122row4_col2,#T_64592914_8c31_11eb_a67f_acde48001122row10_col1{\n",
       "            background-color:  #08478d;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row4_col1,#T_64592914_8c31_11eb_a67f_acde48001122row5_col0,#T_64592914_8c31_11eb_a67f_acde48001122row9_col0{\n",
       "            background-color:  #084488;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row5_col1,#T_64592914_8c31_11eb_a67f_acde48001122row8_col0,#T_64592914_8c31_11eb_a67f_acde48001122row9_col1,#T_64592914_8c31_11eb_a67f_acde48001122row12_col0{\n",
       "            background-color:  #084184;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row5_col2,#T_64592914_8c31_11eb_a67f_acde48001122row6_col1,#T_64592914_8c31_11eb_a67f_acde48001122row9_col2{\n",
       "            background-color:  #08458a;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row6_col0{\n",
       "            background-color:  #084a91;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row6_col2,#T_64592914_8c31_11eb_a67f_acde48001122row10_col0{\n",
       "            background-color:  #084b93;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row7_col0,#T_64592914_8c31_11eb_a67f_acde48001122row7_col2{\n",
       "            background-color:  #083d7f;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row7_col1,#T_64592914_8c31_11eb_a67f_acde48001122row13_col0,#T_64592914_8c31_11eb_a67f_acde48001122row13_col2{\n",
       "            background-color:  #083c7d;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row8_col1,#T_64592914_8c31_11eb_a67f_acde48001122row12_col1{\n",
       "            background-color:  #083e81;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row8_col2,#T_64592914_8c31_11eb_a67f_acde48001122row12_col2{\n",
       "            background-color:  #084082;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row10_col2{\n",
       "            background-color:  #084c95;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row11_col0,#T_64592914_8c31_11eb_a67f_acde48001122row11_col1,#T_64592914_8c31_11eb_a67f_acde48001122row11_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row13_col1{\n",
       "            background-color:  #083979;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_64592914_8c31_11eb_a67f_acde48001122row14_col0,#T_64592914_8c31_11eb_a67f_acde48001122row14_col1,#T_64592914_8c31_11eb_a67f_acde48001122row14_col2,#T_64592914_8c31_11eb_a67f_acde48001122row15_col0,#T_64592914_8c31_11eb_a67f_acde48001122row15_col1,#T_64592914_8c31_11eb_a67f_acde48001122row15_col2{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_64592914_8c31_11eb_a67f_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Standart</th>        <th class=\"col_heading level0 col1\" >LowFP</th>        <th class=\"col_heading level0 col2\" >LowFN</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row0_col0\" class=\"data row0 col0\" >19.770000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row0_col1\" class=\"data row0 col1\" >17.040000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row0_col2\" class=\"data row0 col2\" >20.870000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row1_col0\" class=\"data row1 col0\" >17.710000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row1_col1\" class=\"data row1 col1\" >15.010000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row1_col2\" class=\"data row1 col2\" >18.990000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row2_col0\" class=\"data row2 col0\" >19.330000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row2_col1\" class=\"data row2 col1\" >16.670000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row2_col2\" class=\"data row2 col2\" >20.830000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row3_col0\" class=\"data row3 col0\" >19.770000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row3_col1\" class=\"data row3 col1\" >17.040000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row3_col2\" class=\"data row3 col2\" >20.870000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row4_col0\" class=\"data row4 col0\" >20.520000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row4_col1\" class=\"data row4 col1\" >18.090000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row4_col2\" class=\"data row4 col2\" >21.880000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row5_col0\" class=\"data row5 col0\" >20.890000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row5_col1\" class=\"data row5 col1\" >18.460000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row5_col2\" class=\"data row5 col2\" >22.130000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row6_col0\" class=\"data row6 col0\" >20.250000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row6_col1\" class=\"data row6 col1\" >17.950000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row6_col2\" class=\"data row6 col2\" >21.450000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row7_col0\" class=\"data row7 col0\" >21.530000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row7_col1\" class=\"data row7 col1\" >18.980000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row7_col2\" class=\"data row7 col2\" >22.820000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row8_col0\" class=\"data row8 col0\" >21.240000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row8_col1\" class=\"data row8 col1\" >18.770000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row8_col2\" class=\"data row8 col2\" >22.620000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row9_col0\" class=\"data row9 col0\" >20.890000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row9_col1\" class=\"data row9 col1\" >18.460000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row9_col2\" class=\"data row9 col2\" >22.130000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row10_col0\" class=\"data row10 col0\" >20.160000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row10_col1\" class=\"data row10 col1\" >17.780000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row10_col2\" class=\"data row10 col2\" >21.390000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row11_col0\" class=\"data row11 col0\" >23.070000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row11_col1\" class=\"data row11 col1\" >20.520000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row11_col2\" class=\"data row11 col2\" >24.350000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row12_col0\" class=\"data row12 col0\" >21.200000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row12_col1\" class=\"data row12 col1\" >18.690000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row12_col2\" class=\"data row12 col2\" >22.600000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row13_col0\" class=\"data row13 col0\" >21.700000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row13_col1\" class=\"data row13 col1\" >19.320000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row13_col2\" class=\"data row13 col2\" >22.930000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row14_col0\" class=\"data row14 col0\" >-5.500000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row14_col1\" class=\"data row14 col1\" >-11.000000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row14_col2\" class=\"data row14 col2\" >-3.670000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_64592914_8c31_11eb_a67f_acde48001122level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row15_col0\" class=\"data row15 col0\" >-5.500000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row15_col1\" class=\"data row15 col1\" >-11.000000</td>\n",
       "                        <td id=\"T_64592914_8c31_11eb_a67f_acde48001122row15_col2\" class=\"data row15 col2\" >-3.670000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1369f1070>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(table1).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4 Win experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 Individual algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = (\n",
    "    {'cost':'ar', 'params':{'order':1}, 'width':10},\n",
    "    {'cost':'ar', 'params':{'order':1}, 'width':15},\n",
    "    {'cost':'ar', 'params':{'order':5}, 'width':20},\n",
    "    {'cost':'mahalanobis', 'params':{}, 'width':10},\n",
    "    {'cost':'mahalanobis', 'params':{}, 'width':15},\n",
    "    {'cost':'mahalanobis', 'params':{}, 'width':20},\n",
    "    {'cost':'l1', 'params':{}, 'width':10},\n",
    "    {'cost':'l1', 'params':{}, 'width':15},\n",
    "    {'cost':'l1', 'params':{}, 'width':20},\n",
    "    {'cost':'l2', 'params':{}, 'width':10},\n",
    "    {'cost':'l2', 'params':{}, 'width':15},\n",
    "    {'cost':'l2', 'params':{}, 'width':20},\n",
    "#     {'cost':'linear', 'params':{}, 'width':10},\n",
    "#     {'cost':'linear', 'params':{}, 'width':40},\n",
    "#     {'cost':'linear', 'params':{}, 'width':100}\n",
    "#     {'model':'rbf', 'params':{}, 'width':40},\n",
    "#     {'model':'rbf', 'params':{}, 'width':100},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wind(cost, data, params, width):\n",
    "    predicted_cp = []\n",
    "    for dataset in data:\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(dataset.drop('changepoint', axis=1))\n",
    "        algo = rpt.Window(model=cost, \n",
    "                          params=params, \n",
    "                          width=width,\n",
    "                          jump=1)\n",
    "        algo.fit(signal)\n",
    "        my_bkps = algo.predict(n_bkps=len(dataset[dataset['changepoint']==1]))\n",
    "        \n",
    "        single_predicted_cp = pd.Series(data=0, index=dataset.index)\n",
    "        single_predicted_cp[single_predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "        predicted_cp.append(single_predicted_cp)\n",
    "\n",
    "    true_cp = [dataset.changepoint for dataset in data]\n",
    "    nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')\n",
    "    return nab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 23s, sys: 6.69 s, total: 1min 30s\n",
      "Wall time: 54.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "table = []\n",
    "for model in tqdm_notebook(models, desc='agg functions loop'):\n",
    "    table.append(wind(**model, data=test))\n",
    "    clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_a4b45864_8da6_11eb_a67f_acde48001122row0_col0{\n",
       "            background-color:  #2c7cba;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row0_col1,#T_a4b45864_8da6_11eb_a67f_acde48001122row6_col2{\n",
       "            background-color:  #3585bf;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row0_col2{\n",
       "            background-color:  #2070b4;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row1_col0,#T_a4b45864_8da6_11eb_a67f_acde48001122row1_col1{\n",
       "            background-color:  #d6e6f4;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row1_col2{\n",
       "            background-color:  #d8e7f5;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row2_col0{\n",
       "            background-color:  #e3eef8;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row2_col1{\n",
       "            background-color:  #eef5fc;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row2_col2{\n",
       "            background-color:  #e0ecf8;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row3_col0{\n",
       "            background-color:  #c7dbef;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row3_col1{\n",
       "            background-color:  #cde0f1;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row3_col2{\n",
       "            background-color:  #c4daee;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row4_col0{\n",
       "            background-color:  #d2e3f3;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row4_col1{\n",
       "            background-color:  #d1e2f3;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row4_col2{\n",
       "            background-color:  #d4e4f4;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row5_col0{\n",
       "            background-color:  #529dcc;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row5_col1{\n",
       "            background-color:  #4997c9;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row5_col2{\n",
       "            background-color:  #5fa6d1;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row6_col0{\n",
       "            background-color:  #3a8ac2;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row6_col1{\n",
       "            background-color:  #4896c8;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row7_col0{\n",
       "            background-color:  #105ba4;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row7_col1{\n",
       "            background-color:  #115ca5;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row7_col2{\n",
       "            background-color:  #0e59a2;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row8_col0,#T_a4b45864_8da6_11eb_a67f_acde48001122row8_col1,#T_a4b45864_8da6_11eb_a67f_acde48001122row8_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row9_col0{\n",
       "            background-color:  #b0d2e7;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row9_col1{\n",
       "            background-color:  #c1d9ed;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row9_col2{\n",
       "            background-color:  #a6cee4;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row10_col0,#T_a4b45864_8da6_11eb_a67f_acde48001122row10_col1,#T_a4b45864_8da6_11eb_a67f_acde48001122row10_col2{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row11_col0{\n",
       "            background-color:  #7ab6d9;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row11_col1{\n",
       "            background-color:  #7db8da;\n",
       "            color:  #000000;\n",
       "        }#T_a4b45864_8da6_11eb_a67f_acde48001122row11_col2{\n",
       "            background-color:  #6dafd7;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_a4b45864_8da6_11eb_a67f_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Standart</th>        <th class=\"col_heading level0 col1\" >LowFP</th>        <th class=\"col_heading level0 col2\" >LowFN</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row0_col0\" class=\"data row0 col0\" >16.450000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row0_col1\" class=\"data row0 col1\" >13.930000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row0_col2\" class=\"data row0 col2\" >17.640000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row1_col0\" class=\"data row1 col0\" >12.770000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row1_col1\" class=\"data row1 col1\" >10.380000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row1_col2\" class=\"data row1 col2\" >13.900000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row2_col0\" class=\"data row2 col0\" >12.360000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row2_col1\" class=\"data row2 col1\" >9.580000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row2_col2\" class=\"data row2 col2\" >13.620000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row3_col0\" class=\"data row3 col0\" >13.330000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row3_col1\" class=\"data row3 col1\" >10.720000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row3_col2\" class=\"data row3 col2\" >14.520000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row4_col0\" class=\"data row4 col0\" >12.930000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row4_col1\" class=\"data row4 col1\" >10.590000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row4_col2\" class=\"data row4 col2\" >14.010000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row5_col0\" class=\"data row5 col0\" >15.550000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row5_col1\" class=\"data row5 col1\" >13.440000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row5_col2\" class=\"data row5 col2\" >16.270000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row6_col0\" class=\"data row6 col0\" >16.070000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row6_col1\" class=\"data row6 col1\" >13.470000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row6_col2\" class=\"data row6 col2\" >17.130000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row7_col0\" class=\"data row7 col0\" >17.290000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row7_col1\" class=\"data row7 col1\" >15.030000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row7_col2\" class=\"data row7 col2\" >18.190000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row8_col0\" class=\"data row8 col0\" >18.400000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row8_col1\" class=\"data row8 col1\" >16.220000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row8_col2\" class=\"data row8 col2\" >19.190000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row9_col0\" class=\"data row9 col0\" >13.800000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row9_col1\" class=\"data row9 col1\" >11.090000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row9_col2\" class=\"data row9 col2\" >15.100000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row10_col0\" class=\"data row10 col0\" >11.660000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row10_col1\" class=\"data row10 col1\" >9.230000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row10_col2\" class=\"data row10 col2\" >12.900000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_a4b45864_8da6_11eb_a67f_acde48001122level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row11_col0\" class=\"data row11 col0\" >14.780000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row11_col1\" class=\"data row11 col1\" >12.400000</td>\n",
       "                        <td id=\"T_a4b45864_8da6_11eb_a67f_acde48001122row11_col2\" class=\"data row11 col2\" >16.010000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1356aa790>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(table).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def windowEnsemble(cost, data, num_agg_func, width):\n",
    "    predicted_cp = []\n",
    "    for dataset in data:\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(dataset.drop('changepoint', axis=1))\n",
    "        algo = rpt.WindowEnsembling(custom_cost=cost, \n",
    "                                    jump=1, \n",
    "                                    ensembling=num_agg_func, \n",
    "                                    width=width)\n",
    "        single_predicted_cp = pd.Series(data=0, index=dataset.index)\n",
    "        try:\n",
    "            algo.fit(signal)\n",
    "            my_bkps = algo.predict(n_bkps=len(dataset[dataset['changepoint']==1]))\n",
    "\n",
    "            single_predicted_cp[single_predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "            predicted_cp.append(single_predicted_cp)\n",
    "        except:\n",
    "            predicted_cp.append(single_predicted_cp)\n",
    "    \n",
    "    true_cp = [dataset.changepoint for dataset in data]\n",
    "    nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')\n",
    "    return nab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 30min 30s, sys: 2min 47s, total: 33min 18s\n",
      "Wall time: 16min 49s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cost = CostNew()\n",
    "table1 = []\n",
    "\n",
    "for n in tnrange(1, NUM_CPDE, desc='agg functions loop'):\n",
    "    for w in tqdm_notebook([10, 20, 30], desc='width loop', leave=False):\n",
    "        table1.append(windowEnsemble(cost=cost, data=test, num_agg_func=n, width=w))\n",
    "        clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_cc468b88_8c69_11eb_a67f_acde48001122row0_col0{\n",
       "            background-color:  #2e7ebc;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row0_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row21_col2{\n",
       "            background-color:  #3080bd;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row0_col2{\n",
       "            background-color:  #2b7bba;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row1_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row28_col1{\n",
       "            background-color:  #4594c7;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row1_col1{\n",
       "            background-color:  #4090c5;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row1_col2{\n",
       "            background-color:  #4695c8;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row2_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row10_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row35_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row35_col1{\n",
       "            background-color:  #1e6db2;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row2_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row7_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row7_col1{\n",
       "            background-color:  #1b69af;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row2_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row23_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row35_col2{\n",
       "            background-color:  #206fb4;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row3_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row20_col0{\n",
       "            background-color:  #2979b9;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row3_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row20_col2{\n",
       "            background-color:  #2d7dbb;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row3_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row4_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row16_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row29_col0{\n",
       "            background-color:  #2777b8;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row4_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row4_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row10_col1{\n",
       "            background-color:  #2676b8;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row5_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row39_col0{\n",
       "            background-color:  #3b8bc2;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row5_col1{\n",
       "            background-color:  #3686c0;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row5_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row32_col1{\n",
       "            background-color:  #3f8fc5;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row6_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row34_col0{\n",
       "            background-color:  #6aaed6;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row6_col1{\n",
       "            background-color:  #71b1d7;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row6_col2{\n",
       "            background-color:  #68acd5;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row7_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row17_col0{\n",
       "            background-color:  #1967ad;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row8_col0{\n",
       "            background-color:  #0e59a2;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row8_col1{\n",
       "            background-color:  #0f5aa3;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row8_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row26_col2{\n",
       "            background-color:  #115ca5;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row9_col0{\n",
       "            background-color:  #56a0ce;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row9_col1{\n",
       "            background-color:  #5aa2cf;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row9_col2{\n",
       "            background-color:  #529dcc;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row10_col0{\n",
       "            background-color:  #2272b6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row11_col0{\n",
       "            background-color:  #5ca4d0;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row11_col1{\n",
       "            background-color:  #57a0ce;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row11_col2{\n",
       "            background-color:  #64a9d3;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row12_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row28_col0{\n",
       "            background-color:  #4d99ca;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row12_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row22_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row22_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row22_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row32_col0{\n",
       "            background-color:  #4997c9;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row12_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row39_col2{\n",
       "            background-color:  #4b98ca;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row13_col0{\n",
       "            background-color:  #1865ac;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row13_col1{\n",
       "            background-color:  #125ea6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row13_col2{\n",
       "            background-color:  #1c6ab0;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row14_col0{\n",
       "            background-color:  #1a68ae;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row14_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row17_col1{\n",
       "            background-color:  #1562a9;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row14_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row23_col0{\n",
       "            background-color:  #1d6cb1;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row15_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row15_col2{\n",
       "            background-color:  #60a7d2;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row15_col1{\n",
       "            background-color:  #5ba3d0;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row16_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row29_col1{\n",
       "            background-color:  #2070b4;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row16_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row17_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row23_col1{\n",
       "            background-color:  #1c6bb0;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row18_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row42_col0{\n",
       "            background-color:  #a4cce3;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row18_col1{\n",
       "            background-color:  #9ac8e0;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row18_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row40_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row42_col2{\n",
       "            background-color:  #a6cee4;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row19_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row19_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row19_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row20_col1{\n",
       "            background-color:  #2373b6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row21_col0{\n",
       "            background-color:  #3585bf;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row21_col1{\n",
       "            background-color:  #3c8cc3;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row24_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row27_col2{\n",
       "            background-color:  #7fb9da;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row24_col1{\n",
       "            background-color:  #7cb7da;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row24_col2{\n",
       "            background-color:  #7db8da;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row25_col0{\n",
       "            background-color:  #3282be;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row25_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row29_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row36_col0{\n",
       "            background-color:  #2c7cba;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row25_col2{\n",
       "            background-color:  #3989c1;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row26_col0{\n",
       "            background-color:  #0e58a2;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row26_col1{\n",
       "            background-color:  #0a549e;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row27_col0{\n",
       "            background-color:  #82bbdb;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row27_col1{\n",
       "            background-color:  #81badb;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row28_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row33_col2{\n",
       "            background-color:  #539ecd;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row30_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row30_col2{\n",
       "            background-color:  #95c5df;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row30_col1{\n",
       "            background-color:  #92c4de;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row31_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row31_col1{\n",
       "            background-color:  #084082;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row31_col2{\n",
       "            background-color:  #084184;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row32_col2{\n",
       "            background-color:  #4f9bcb;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row33_col0{\n",
       "            background-color:  #58a1cf;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row33_col1{\n",
       "            background-color:  #5da5d1;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row34_col1{\n",
       "            background-color:  #66abd4;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row34_col2{\n",
       "            background-color:  #6fb0d7;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row36_col1{\n",
       "            background-color:  #0c56a0;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row36_col2{\n",
       "            background-color:  #3a8ac2;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row37_col0{\n",
       "            background-color:  #c9ddf0;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row37_col1{\n",
       "            background-color:  #b5d4e9;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row37_col2{\n",
       "            background-color:  #cbdef1;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row38_col0{\n",
       "            background-color:  #aacfe5;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row38_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row38_col2{\n",
       "            background-color:  #abd0e6;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row39_col1{\n",
       "            background-color:  #1764ab;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row40_col0{\n",
       "            background-color:  #bdd7ec;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row40_col2,#T_cc468b88_8c69_11eb_a67f_acde48001122row43_col1{\n",
       "            background-color:  #bed8ec;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row41_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row41_col1,#T_cc468b88_8c69_11eb_a67f_acde48001122row41_col2{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row42_col1{\n",
       "            background-color:  #9cc9e1;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row43_col0{\n",
       "            background-color:  #bfd8ed;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row43_col2{\n",
       "            background-color:  #c6dbef;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row44_col0{\n",
       "            background-color:  #b4d3e9;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row44_col1{\n",
       "            background-color:  #b3d3e8;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row44_col2{\n",
       "            background-color:  #b2d2e8;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row45_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row45_col2{\n",
       "            background-color:  #d1e2f3;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row45_col1{\n",
       "            background-color:  #cfe1f2;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row46_col0,#T_cc468b88_8c69_11eb_a67f_acde48001122row46_col2{\n",
       "            background-color:  #a9cfe5;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row46_col1{\n",
       "            background-color:  #a5cde3;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row47_col0{\n",
       "            background-color:  #d9e8f5;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row47_col1{\n",
       "            background-color:  #dce9f6;\n",
       "            color:  #000000;\n",
       "        }#T_cc468b88_8c69_11eb_a67f_acde48001122row47_col2{\n",
       "            background-color:  #d6e6f4;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_cc468b88_8c69_11eb_a67f_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Standart</th>        <th class=\"col_heading level0 col1\" >LowFP</th>        <th class=\"col_heading level0 col2\" >LowFN</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row0_col0\" class=\"data row0 col0\" >15.530000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row0_col1\" class=\"data row0 col1\" >12.760000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row0_col2\" class=\"data row0 col2\" >16.760000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row1_col0\" class=\"data row1 col0\" >14.410000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row1_col1\" class=\"data row1 col1\" >11.880000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row1_col2\" class=\"data row1 col2\" >15.510000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row2_col0\" class=\"data row2 col0\" >16.340000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row2_col1\" class=\"data row2 col1\" >13.960000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row2_col2\" class=\"data row2 col2\" >17.300000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row3_col0\" class=\"data row3 col0\" >15.740000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row3_col1\" class=\"data row3 col1\" >12.910000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row3_col2\" class=\"data row3 col2\" >16.900000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row4_col0\" class=\"data row4 col0\" >15.850000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row4_col1\" class=\"data row4 col1\" >13.190000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row4_col2\" class=\"data row4 col2\" >16.980000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row5_col0\" class=\"data row5 col0\" >14.830000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row5_col1\" class=\"data row5 col1\" >12.410000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row5_col2\" class=\"data row5 col2\" >15.780000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row6_col0\" class=\"data row6 col0\" >12.900000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row6_col1\" class=\"data row6 col1\" >9.830000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row6_col2\" class=\"data row6 col2\" >14.240000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row7_col0\" class=\"data row7 col0\" >16.510000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row7_col1\" class=\"data row7 col1\" >13.920000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row7_col2\" class=\"data row7 col2\" >17.680000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row8_col0\" class=\"data row8 col0\" >17.340000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row8_col1\" class=\"data row8 col1\" >14.790000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row8_col2\" class=\"data row8 col2\" >18.230000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row9_col0\" class=\"data row9 col0\" >13.730000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row9_col1\" class=\"data row9 col1\" >10.780000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row9_col2\" class=\"data row9 col2\" >15.050000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row10_col0\" class=\"data row10 col0\" >16.080000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row10_col1\" class=\"data row10 col1\" >13.220000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row10_col2\" class=\"data row10 col2\" >17.390000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row11_col0\" class=\"data row11 col0\" >13.480000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row11_col1\" class=\"data row11 col1\" >10.890000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row11_col2\" class=\"data row11 col2\" >14.370000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row12_col0\" class=\"data row12 col0\" >14.090000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row12_col1\" class=\"data row12 col1\" >11.480000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row12_col2\" class=\"data row12 col2\" >15.290000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row13_col0\" class=\"data row13 col0\" >16.700000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row13_col1\" class=\"data row13 col1\" >14.540000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row13_col2\" class=\"data row13 col2\" >17.540000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row14_col0\" class=\"data row14 col0\" >16.590000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row14_col1\" class=\"data row14 col1\" >14.310000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row14_col2\" class=\"data row14 col2\" >17.470000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row15_col0\" class=\"data row15 col0\" >13.320000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row15_col1\" class=\"data row15 col1\" >10.710000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row15_col2\" class=\"data row15 col2\" >14.520000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row16\" class=\"row_heading level0 row16\" >16</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row16_col0\" class=\"data row16 col0\" >16.140000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row16_col1\" class=\"data row16 col1\" >13.850000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row16_col2\" class=\"data row16 col2\" >16.910000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row17\" class=\"row_heading level0 row17\" >17</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row17_col0\" class=\"data row17 col0\" >16.600000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row17_col1\" class=\"data row17 col1\" >14.320000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row17_col2\" class=\"data row17 col2\" >17.480000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row18\" class=\"row_heading level0 row18\" >18</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row18_col0\" class=\"data row18 col0\" >11.050000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row18_col1\" class=\"data row18 col1\" >8.410000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row18_col2\" class=\"data row18 col2\" >12.240000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row19\" class=\"row_heading level0 row19\" >19</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row19_col0\" class=\"data row19 col0\" >19.380000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row19_col1\" class=\"data row19 col1\" >17.030000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row19_col2\" class=\"data row19 col2\" >20.350000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row20\" class=\"row_heading level0 row20\" >20</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row20_col0\" class=\"data row20 col0\" >15.750000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row20_col1\" class=\"data row20 col1\" >13.390000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row20_col2\" class=\"data row20 col2\" >16.650000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row21\" class=\"row_heading level0 row21\" >21</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row21_col0\" class=\"data row21 col0\" >15.140000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row21_col1\" class=\"data row21 col1\" >12.070000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row21_col2\" class=\"data row21 col2\" >16.500000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row22\" class=\"row_heading level0 row22\" >22</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row22_col0\" class=\"data row22 col0\" >14.250000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row22_col1\" class=\"data row22 col1\" >11.500000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row22_col2\" class=\"data row22 col2\" >15.390000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row23\" class=\"row_heading level0 row23\" >23</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row23_col0\" class=\"data row23 col0\" >16.350000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row23_col1\" class=\"data row23 col1\" >13.820000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row23_col2\" class=\"data row23 col2\" >17.310000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row24\" class=\"row_heading level0 row24\" >24</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row24_col0\" class=\"data row24 col0\" >12.250000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row24_col1\" class=\"data row24 col1\" >9.460000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row24_col2\" class=\"data row24 col2\" >13.550000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row25\" class=\"row_heading level0 row25\" >25</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row25_col0\" class=\"data row25 col0\" >15.310000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row25_col1\" class=\"data row25 col1\" >12.940000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row25_col2\" class=\"data row25 col2\" >16.100000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row26\" class=\"row_heading level0 row26\" >26</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row26_col0\" class=\"data row26 col0\" >17.370000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row26_col1\" class=\"data row26 col1\" >15.090000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row26_col2\" class=\"data row26 col2\" >18.250000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row27\" class=\"row_heading level0 row27\" >27</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row27_col0\" class=\"data row27 col0\" >12.160000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row27_col1\" class=\"data row27 col1\" >9.330000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row27_col2\" class=\"data row27 col2\" >13.490000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row28\" class=\"row_heading level0 row28\" >28</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row28_col0\" class=\"data row28 col0\" >14.080000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row28_col1\" class=\"data row28 col1\" >11.630000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row28_col2\" class=\"data row28 col2\" >15.030000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row29\" class=\"row_heading level0 row29\" >29</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row29_col0\" class=\"data row29 col0\" >15.830000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row29_col1\" class=\"data row29 col1\" >13.550000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row29_col2\" class=\"data row29 col2\" >16.710000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row30\" class=\"row_heading level0 row30\" >30</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row30_col0\" class=\"data row30 col0\" >11.560000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row30_col1\" class=\"data row30 col1\" >8.710000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row30_col2\" class=\"data row30 col2\" >12.840000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row31\" class=\"row_heading level0 row31\" >31</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row31_col0\" class=\"data row31 col0\" >18.580000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row31_col1\" class=\"data row31 col1\" >16.190000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row31_col2\" class=\"data row31 col2\" >19.570000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row32\" class=\"row_heading level0 row32\" >32</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row32_col0\" class=\"data row32 col0\" >14.250000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row32_col1\" class=\"data row32 col1\" >11.930000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row32_col2\" class=\"data row32 col2\" >15.140000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row33\" class=\"row_heading level0 row33\" >33</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row33_col0\" class=\"data row33 col0\" >13.640000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row33_col1\" class=\"data row33 col1\" >10.600000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row33_col2\" class=\"data row33 col2\" >14.990000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row34\" class=\"row_heading level0 row34\" >34</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row34_col0\" class=\"data row34 col0\" >12.900000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row34_col1\" class=\"data row34 col1\" >10.220000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row34_col2\" class=\"data row34 col2\" >13.990000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row35\" class=\"row_heading level0 row35\" >35</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row35_col0\" class=\"data row35 col0\" >16.310000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row35_col1\" class=\"data row35 col1\" >13.730000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row35_col2\" class=\"data row35 col2\" >17.280000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row36\" class=\"row_heading level0 row36\" >36</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row36_col0\" class=\"data row36 col0\" >15.630000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row36_col1\" class=\"data row36 col1\" >14.970000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row36_col2\" class=\"data row36 col2\" >16.060000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row37\" class=\"row_heading level0 row37\" >37</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row37_col0\" class=\"data row37 col0\" >9.500000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row37_col1\" class=\"data row37 col1\" >7.310000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row37_col2\" class=\"data row37 col2\" >10.690000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row38\" class=\"row_heading level0 row38\" >38</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row38_col0\" class=\"data row38 col0\" >10.790000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row38_col1\" class=\"data row38 col1\" >7.700000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row38_col2\" class=\"data row38 col2\" >12.060000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row39\" class=\"row_heading level0 row39\" >39</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row39_col0\" class=\"data row39 col0\" >14.860000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row39_col1\" class=\"data row39 col1\" >14.200000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row39_col2\" class=\"data row39 col2\" >15.290000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row40\" class=\"row_heading level0 row40\" >40</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row40_col0\" class=\"data row40 col0\" >10.060000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row40_col1\" class=\"data row40 col1\" >7.930000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row40_col2\" class=\"data row40 col2\" >11.320000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row41\" class=\"row_heading level0 row41\" >41</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row41_col0\" class=\"data row41 col0\" >6.420000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row41_col1\" class=\"data row41 col1\" >3.060000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row41_col2\" class=\"data row41 col2\" >7.870000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row42\" class=\"row_heading level0 row42\" >42</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row42_col0\" class=\"data row42 col0\" >11.070000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row42_col1\" class=\"data row42 col1\" >8.400000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row42_col2\" class=\"data row42 col2\" >12.250000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row43\" class=\"row_heading level0 row43\" >43</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row43_col0\" class=\"data row43 col0\" >9.960000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row43_col1\" class=\"data row43 col1\" >6.910000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row43_col2\" class=\"data row43 col2\" >11.000000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row44\" class=\"row_heading level0 row44\" >44</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row44_col0\" class=\"data row44 col0\" >10.380000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row44_col1\" class=\"data row44 col1\" >7.410000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row44_col2\" class=\"data row44 col2\" >11.790000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row45\" class=\"row_heading level0 row45\" >45</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row45_col0\" class=\"data row45 col0\" >8.910000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row45_col1\" class=\"data row45 col1\" >5.950000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row45_col2\" class=\"data row45 col2\" >10.300000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row46\" class=\"row_heading level0 row46\" >46</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row46_col0\" class=\"data row46 col0\" >10.870000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row46_col1\" class=\"data row46 col1\" >7.980000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row46_col2\" class=\"data row46 col2\" >12.120000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_cc468b88_8c69_11eb_a67f_acde48001122level0_row47\" class=\"row_heading level0 row47\" >47</th>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row47_col0\" class=\"data row47 col0\" >8.390000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row47_col1\" class=\"data row47 col1\" >4.990000</td>\n",
       "                        <td id=\"T_cc468b88_8c69_11eb_a67f_acde48001122row47_col2\" class=\"data row47 col2\" >9.950000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x134c86ca0>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(table1).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5 BinSeg experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.1 Individual algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = (\n",
    "    {'cost':'ar', 'params':{'order':1}},\n",
    "    {'cost':'mahalanobis', 'params':{}},\n",
    "    {'cost':'l1', 'params':{}},\n",
    "    {'cost':'l2', 'params':{}},\n",
    "    {'cost':'linear', 'params':{}},\n",
    "#     {'model':'rbf', 'params':{}},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binseg(cost, data, params):\n",
    "    predicted_cp = []\n",
    "    for dataset in data:\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(dataset.drop('changepoint', axis=1))\n",
    "        algo = rpt.Binseg(model=cost, \n",
    "                          params=params,\n",
    "                          jump=5)\n",
    "        algo.fit(signal)\n",
    "        my_bkps = algo.predict(n_bkps=len(dataset[dataset['changepoint']==1]))\n",
    "        \n",
    "        single_predicted_cp = pd.Series(data=0, index=dataset.index)\n",
    "        single_predicted_cp[single_predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "        predicted_cp.append(single_predicted_cp)\n",
    "\n",
    "    true_cp = [dataset.changepoint for dataset in data]\n",
    "    nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')\n",
    "    return nab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = []\n",
    "for model in tqdm_notebook(models, desc='agg functions loop'):\n",
    "    table.append(binseg(**model, data=test))\n",
    "    clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row0_col0,#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row0_col1{\n",
       "            background-color:  #1460a8;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row0_col2{\n",
       "            background-color:  #105ba4;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row1_col0,#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row1_col1,#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row1_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row2_col0{\n",
       "            background-color:  #539ecd;\n",
       "            color:  #000000;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row2_col1{\n",
       "            background-color:  #57a0ce;\n",
       "            color:  #000000;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row2_col2{\n",
       "            background-color:  #529dcc;\n",
       "            color:  #000000;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row3_col0{\n",
       "            background-color:  #5ca4d0;\n",
       "            color:  #000000;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row3_col1{\n",
       "            background-color:  #5da5d1;\n",
       "            color:  #000000;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row3_col2{\n",
       "            background-color:  #58a1cf;\n",
       "            color:  #000000;\n",
       "        }#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row4_col0,#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row4_col1,#T_dfc0c4cc_8c31_11eb_a67f_acde48001122row4_col2{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Standart</th>        <th class=\"col_heading level0 col1\" >LowFP</th>        <th class=\"col_heading level0 col2\" >LowFN</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row0_col0\" class=\"data row0 col0\" >21.390000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row0_col1\" class=\"data row0 col1\" >18.890000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row0_col2\" class=\"data row0 col2\" >22.720000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row1_col0\" class=\"data row1 col0\" >24.100000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row1_col1\" class=\"data row1 col1\" >21.690000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row1_col2\" class=\"data row1 col2\" >25.040000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row2_col0\" class=\"data row2 col0\" >17.870000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row2_col1\" class=\"data row2 col1\" >15.100000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row2_col2\" class=\"data row2 col2\" >19.090000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row3_col0\" class=\"data row3 col0\" >17.460000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row3_col1\" class=\"data row3 col1\" >14.810000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row3_col2\" class=\"data row3 col2\" >18.820000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row4_col0\" class=\"data row4 col0\" >9.530000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row4_col1\" class=\"data row4 col1\" >6.700000</td>\n",
       "                        <td id=\"T_dfc0c4cc_8c31_11eb_a67f_acde48001122row4_col2\" class=\"data row4 col2\" >10.970000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1393ddf10>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(table).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5.2 Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binsegEnsemble(cost, data, num_agg_func):\n",
    "    predicted_cp = []\n",
    "    for dataset in data:\n",
    "        stsc = StandardScaler()\n",
    "        signal = stsc.fit_transform(dataset.drop('changepoint', axis=1))\n",
    "        algo = rpt.BinsegEnsembling(custom_cost=cost, \n",
    "                                    jump=5, \n",
    "                                    ensembling=num_agg_func)\n",
    "        single_predicted_cp = pd.Series(data=0, index=dataset.index)\n",
    "        try:\n",
    "            algo.fit(signal)\n",
    "            my_bkps = algo.predict(n_bkps=len(dataset[dataset['changepoint']==1]))\n",
    "\n",
    "            single_predicted_cp[single_predicted_cp.index[my_bkps[:-1]]] = 1\n",
    "            predicted_cp.append(single_predicted_cp)\n",
    "        except:\n",
    "            predicted_cp.append(single_predicted_cp)\n",
    "    \n",
    "    true_cp = [dataset.changepoint for dataset in data]\n",
    "    nab = evaluating_change_point(true_cp, predicted_cp, metric='nab', numenta_time='30 sec')\n",
    "    return nab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12min 30s, sys: 1min 7s, total: 13min 37s\n",
      "Wall time: 6min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cost = CostNew()\n",
    "table1 = []\n",
    "\n",
    "for n in tnrange(1, NUM_CPDE, desc='agg functions loop'):\n",
    "    table1.append(binsegEnsemble(cost, data=test, num_agg_func=n))\n",
    "    clear_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "#T_c46bc6b2_8c32_11eb_a67f_acde48001122row0_col0{\n",
       "            background-color:  #f5fafe;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row0_col1,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row15_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row15_col2{\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row0_col2{\n",
       "            background-color:  #e3eef9;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row1_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row1_col2{\n",
       "            background-color:  #2979b9;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row1_col1{\n",
       "            background-color:  #1c6ab0;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row2_col0{\n",
       "            background-color:  #99c7e0;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row2_col1{\n",
       "            background-color:  #87bddc;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row2_col2{\n",
       "            background-color:  #84bcdb;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row3_col0{\n",
       "            background-color:  #f1f7fd;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row3_col1{\n",
       "            background-color:  #eff6fc;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row3_col2{\n",
       "            background-color:  #e1edf8;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row4_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row6_col2,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row9_col1,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row12_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row14_col2{\n",
       "            background-color:  #09529d;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row4_col1,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row12_col1{\n",
       "            background-color:  #084f99;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row4_col2,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row12_col2{\n",
       "            background-color:  #0a539e;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row5_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row13_col0{\n",
       "            background-color:  #3d8dc4;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row5_col1,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row13_col1{\n",
       "            background-color:  #2272b6;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row5_col2,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row13_col2{\n",
       "            background-color:  #4292c6;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row6_col0{\n",
       "            background-color:  #08509b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row6_col1{\n",
       "            background-color:  #084b93;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row7_col0{\n",
       "            background-color:  #57a0ce;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row7_col1{\n",
       "            background-color:  #3c8cc3;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row7_col2{\n",
       "            background-color:  #549fcd;\n",
       "            color:  #000000;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row8_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row10_col0{\n",
       "            background-color:  #08488e;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row8_col1,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row10_col1{\n",
       "            background-color:  #084387;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row8_col2,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row10_col2{\n",
       "            background-color:  #084a91;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row9_col0{\n",
       "            background-color:  #0e58a2;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row9_col2{\n",
       "            background-color:  #0d57a1;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row11_col0,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row11_col1,#T_c46bc6b2_8c32_11eb_a67f_acde48001122row11_col2{\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row14_col0{\n",
       "            background-color:  #08519c;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row14_col1{\n",
       "            background-color:  #084c95;\n",
       "            color:  #f1f1f1;\n",
       "        }#T_c46bc6b2_8c32_11eb_a67f_acde48001122row15_col1{\n",
       "            background-color:  #caddf0;\n",
       "            color:  #000000;\n",
       "        }</style><table id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122\" ><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >Standart</th>        <th class=\"col_heading level0 col1\" >LowFP</th>        <th class=\"col_heading level0 col2\" >LowFN</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row0_col0\" class=\"data row0 col0\" >0.180000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row0_col1\" class=\"data row0 col1\" >-4.690000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row0_col2\" class=\"data row0 col2\" >1.910000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row1_col0\" class=\"data row1 col0\" >13.030000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row1_col1\" class=\"data row1 col1\" >10.850000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row1_col2\" class=\"data row1 col2\" >14.070000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row2_col0\" class=\"data row2 col0\" >7.050000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row2_col1\" class=\"data row2 col1\" >3.980000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row2_col2\" class=\"data row2 col2\" >8.550000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row3_col0\" class=\"data row3 col0\" >0.600000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row3_col1\" class=\"data row3 col1\" >-3.840000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row3_col2\" class=\"data row3 col2\" >2.190000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row4_col0\" class=\"data row4 col0\" >15.710000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row4_col1\" class=\"data row4 col1\" >13.000000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row4_col2\" class=\"data row4 col2\" >16.890000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row5_col0\" class=\"data row5 col0\" >11.640000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row5_col1\" class=\"data row5 col1\" >10.240000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row5_col2\" class=\"data row5 col2\" >12.120000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row6_col0\" class=\"data row6 col0\" >15.840000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row6_col1\" class=\"data row6 col1\" >13.260000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row6_col2\" class=\"data row6 col2\" >16.970000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row7_col0\" class=\"data row7 col0\" >10.150000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row7_col1\" class=\"data row7 col1\" >8.290000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row7_col2\" class=\"data row7 col2\" >11.120000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row8_col0\" class=\"data row8 col0\" >16.410000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row8_col1\" class=\"data row8 col1\" >13.950000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row8_col2\" class=\"data row8 col2\" >17.600000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row9_col0\" class=\"data row9 col0\" >15.300000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row9_col1\" class=\"data row9 col1\" >12.740000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row9_col2\" class=\"data row9 col2\" >16.610000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row10_col0\" class=\"data row10 col0\" >16.410000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row10_col1\" class=\"data row10 col1\" >13.950000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row10_col2\" class=\"data row10 col2\" >17.600000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row11_col0\" class=\"data row11 col0\" >18.100000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row11_col1\" class=\"data row11 col1\" >15.360000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row11_col2\" class=\"data row11 col2\" >19.510000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row12_col0\" class=\"data row12 col0\" >15.710000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row12_col1\" class=\"data row12 col1\" >13.000000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row12_col2\" class=\"data row12 col2\" >16.890000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row13_col0\" class=\"data row13 col0\" >11.640000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row13_col1\" class=\"data row13 col1\" >10.240000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row13_col2\" class=\"data row13 col2\" >12.120000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row14_col0\" class=\"data row14 col0\" >15.800000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row14_col1\" class=\"data row14 col1\" >13.170000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row14_col2\" class=\"data row14 col2\" >16.940000</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row15_col0\" class=\"data row15 col0\" >0.000000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row15_col1\" class=\"data row15 col1\" >0.000000</td>\n",
       "                        <td id=\"T_c46bc6b2_8c32_11eb_a67f_acde48001122row15_col2\" class=\"data row15 col2\" >0.000000</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x1373cb4f0>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(table1).style.background_gradient(cmap='Blues')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}