{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compare STree with different estimators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup\n",
    "Uncomment the next cell if STree is not already installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# Google Colab setup\n",
    "#\n",
    "#!pip install git+https://github.com/doctorado-ml/stree\n",
    "!pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime, time\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix, f1_score\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from stree import Stree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('data/creditcard.csv'):\n",
    "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
    "    !tar xzf creditcard.tgz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(datetime.date.today(), time.strftime(\"%H:%M:%S\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load dataset and normalize values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Dataset\n",
    "df = pd.read_csv('data/creditcard.csv')\n",
    "df.shape\n",
    "random_state = 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
    "print(\"Valid: {0:.3f}% {1:,}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize Amount\n",
    "from sklearn.preprocessing import RobustScaler\n",
    "values = RobustScaler().fit_transform(df.Amount.values.reshape(-1, 1))\n",
    "df['Amount_Scaled'] = values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Remove unneeded features\n",
    "y = df.Class.values\n",
    "X = df.drop(['Class', 'Time', 'Amount'], axis=1).values\n",
    "print(f\"X shape: {X.shape}\\ny shape: {y.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Divide dataset\n",
    "train_size = .7\n",
    "Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=train_size, shuffle=True, random_state=random_state, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear Tree\n",
    "linear_tree = DecisionTreeClassifier(random_state=random_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Naive Bayes\n",
    "naive_bayes = GaussianNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stree\n",
    "stree = Stree(random_state=random_state, C=.01, max_iter=1000, kernel=\"liblinear\", multiclass_strategy=\"ovr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Neural Network\n",
    "mlp = MLPClassifier(random_state=random_state, alpha=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVC (linear)\n",
    "svc = LinearSVC(random_state=random_state, C=.01, max_iter=1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Do the test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_model(name, model):\n",
    "    print(f\"************************** {name} **********************\")\n",
    "    now = time.time()\n",
    "    model.fit(Xtrain, ytrain)\n",
    "    spent = time.time() - now\n",
    "    print(f\"Train Model {name} took: {spent:.4} seconds\")\n",
    "    predict = model.predict(Xtrain)\n",
    "    predictt = model.predict(Xtest)\n",
    "    print(f\"=========== {name} - Train {Xtrain.shape[0]:,} samples =============\",)\n",
    "    print(classification_report(ytrain, predict, digits=6))\n",
    "    print(f\"=========== {name} - Test {Xtest.shape[0]:,} samples =============\")\n",
    "    print(classification_report(ytest, predictt, digits=6))\n",
    "    print(\"Confusion Matrix in Train\")\n",
    "    print(confusion_matrix(ytrain, predict))\n",
    "    print(\"Confusion Matrix in Test\")\n",
    "    print(confusion_matrix(ytest, predictt))\n",
    "    return f1_score(ytest, predictt), spent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Train & Test models\n",
    "models = {\n",
    "    'Linear Tree':linear_tree, 'Naive Bayes': naive_bayes, 'Stree    ': stree,  \n",
    "    'Neural Network': mlp, 'SVC (linear)': svc\n",
    "}\n",
    "\n",
    "best_f1 = 0\n",
    "outcomes = []\n",
    "for name, model in models.items():\n",
    "    f1, time_spent = try_model(name, model)\n",
    "    outcomes.append((name, f1, time_spent))\n",
    "    if f1 > best_f1:\n",
    "        best_model = name\n",
    "        best_time = time_spent\n",
    "        best_f1 = f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\"*\"*110)\n",
    "print(f\"*The best f1 model is {best_model}, with a f1 score: {best_f1:.4} in {best_time:.6} seconds with {train_size:,} samples in train dataset\")\n",
    "print(\"*\"*110)\n",
    "for name, f1, time_spent in outcomes:\n",
    "    print(f\"Model: {name}\\t Time: {time_spent:6.2f} seconds\\t f1: {f1:.4}\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "**************************************************************************************************************\n",
    "*The best f1 model is Stree (SVM Tree), with a f1 score: 0.8603 in 28.4743 seconds with 0.7 samples in train dataset\n",
    "**************************************************************************************************************\n",
    "Model: Linear Tree\t Time:  10.25 seconds\t f1: 0.7645\n",
    "Model: Naive Bayes\t Time:   0.10 seconds\t f1: 0.1154\n",
    "Model: Stree (SVM Tree)\t Time:  28.47 seconds\t f1: 0.8603\n",
    "Model: Neural Network\t Time:   9.76 seconds\t f1: 0.7381\n",
    "Model: SVC (linear)\t Time:   8.21 seconds\t f1: 0.739"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stree.get_params()"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2-final"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "392px",
    "left": "1518px",
    "right": "20px",
    "top": "40px",
    "width": "392px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
