Fix problem with zero weighted samples

Solve WARNING: class label x specified in weight is not found
with a different approach

.coveragerc

@@ -0,0 +1,13 @@
+[run]
+branch = True
+source = stree
+
+[report]
+exclude_lines =
+    if self.debug:
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:
+ignore_errors = True
+omit =
+    stree/__init__.py

.github/workflows/main.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+        python: [3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install dependencies
+        run: |
+          pip install -q --upgrade pip
+          pip install -q -r requirements.txt
+          pip install -q --upgrade codecov coverage black flake8 codacy-coverage
+      - name: Lint
+        run: |
+          black --check --diff stree
+          flake8 --count stree
+      - name: Tests
+        run: |
+          coverage run -m unittest -v stree.tests
+          coverage xml
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+      - name: Run codacy-coverage-reporter
+        if: runner.os == 'Linux'
+        uses: codacy/codacy-coverage-reporter-action@master
+        with:
+          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+          coverage-reports: coverage.xml

.gitignore

@@ -130,3 +130,7 @@ dmypy.json
 
 .idea
 .vscode
+.pre-commit-config.yaml
+
+**.csv
+.virtual_documents

.travis.yml

@@ -1,13 +0,0 @@
-language: python
-os: linux
-dist: xenial
-install:
-  - pip install -r requirements.txt
-notifications:
-  email:
-    recipients:
-      - ricardo.montanana@alu.uclm.es
-    on_success: never # default: change
-    on_failure: always # default: always
-# command to run tests
-script: python -m unittest tests.Stree_test tests.Snode_test

README.md

@@ -1,23 +1,65 @@
-[![Build Status](https://travis-ci.com/Doctorado-ML/STree.svg?branch=master)](https://travis-ci.com/Doctorado-ML/STree)
+![CI](https://github.com/Doctorado-ML/STree/workflows/CI/badge.svg)
+[![codecov](https://codecov.io/gh/doctorado-ml/stree/branch/master/graph/badge.svg)](https://codecov.io/gh/doctorado-ml/stree)
+[![Codacy Badge](https://app.codacy.com/project/badge/Grade/35fa3dfd53a24a339344b33d9f9f2f3d)](https://www.codacy.com/gh/Doctorado-ML/STree?utm_source=github.com&utm_medium=referral&utm_content=Doctorado-ML/STree&utm_campaign=Badge_Grade)
 
 # Stree
 
-Oblique Tree classifier based on SVM nodes
+Oblique Tree classifier based on SVM nodes. The nodes are built and splitted with sklearn SVC models. Stree is a sklearn estimator and can be integrated in pipelines, grid searches, etc.
 
-## Example
+![Stree](https://raw.github.com/doctorado-ml/stree/master/example.png)
 
-### Jupyter
+## Installation
 
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Doctorado-ML/STree/master?urlpath=lab/tree/test.ipynb)
-
-### Command line
-
-```python
-python main.py
+```bash
+pip install git+https://github.com/doctorado-ml/stree
 ```
 
+## Examples
+
+### Jupyter notebooks
+
+- [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Doctorado-ML/STree/master?urlpath=lab/tree/notebooks/benchmark.ipynb) Benchmark
+
+- [![Test](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/benchmark.ipynb) Benchmark
+
+- [![Test2](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/features.ipynb) Test features
+
+- [![Adaboost](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/adaboost.ipynb) Adaboost
+
+- [![Gridsearch](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/gridsearch.ipynb) Gridsearch
+
+- [![Test Graphics](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/test_graphs.ipynb) Test Graphics
+
+## Hyperparameters
+
+|     | **Hyperparameter** | **Type/Values**                                        | **Default** | **Meaning**                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| --- | ------------------ | ------------------------------------------------------ | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| \*  | C                  | \<float\>                                              | 1.0         | Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive.                                                                                                                                                                                                                                                                                                                              |
+| \*  | kernel             | {"linear", "poly", "rbf"}                              | linear      | Specifies the kernel type to be used in the algorithm. It must be one of ‘linear’, ‘poly’ or ‘rbf’.                                                                                                                                                                                                                                                                                                                                                  |
+| \*  | max_iter           | \<int\>                                                | 1e5         | Hard limit on iterations within solver, or -1 for no limit.                                                                                                                                                                                                                                                                                                                                                                                          |
+| \*  | random_state       | \<int\>                                                | None        | Controls the pseudo random number generation for shuffling the data for probability estimates. Ignored when probability is False.<br>Pass an int for reproducible output across multiple function calls                                                                                                                                                                                                                                              |
+|     | max_depth          | \<int\>                                                | None        | Specifies the maximum depth of the tree                                                                                                                                                                                                                                                                                                                                                                                                              |
+| \*  | tol                | \<float\>                                              | 1e-4        | Tolerance for stopping criterion.                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| \*  | degree             | \<int\>                                                | 3           | Degree of the polynomial kernel function (‘poly’). Ignored by all other kernels.                                                                                                                                                                                                                                                                                                                                                                     |
+| \*  | gamma              | {"scale", "auto"} or \<float\>                         | scale       | Kernel coefficient for ‘rbf’ and ‘poly’.<br>if gamma='scale' (default) is passed then it uses 1 / (n_features \* X.var()) as value of gamma,<br>if ‘auto’, uses 1 / n_features.                                                                                                                                                                                                                                                                      |
+|     | split_criteria     | {"impurity", "max_samples"}                            | impurity    | Decides (just in case of a multi class classification) which column (class) use to split the dataset in a node\*\*                                                                                                                                                                                                                                                                                                                                   |
+|     | criterion          | {“gini”, “entropy”}                                    | entropy     | The function to measure the quality of a split (only used if max_features != num_features). <br>Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain.                                                                                                                                                                                                                                                          |
+|     | min_samples_split  | \<int\>                                                | 0           | The minimum number of samples required to split an internal node. 0 (default) for any                                                                                                                                                                                                                                                                                                                                                                |
+|     | max_features       | \<int\>, \<float\> <br><br>or {“auto”, “sqrt”, “log2”} | None        | The number of features to consider when looking for the split:<br>If int, then consider max_features features at each split.<br>If float, then max_features is a fraction and int(max_features \* n_features) features are considered at each split.<br>If “auto”, then max_features=sqrt(n_features).<br>If “sqrt”, then max_features=sqrt(n_features).<br>If “log2”, then max_features=log2(n_features).<br>If None, then max_features=n_features. |
+|     | splitter           | {"best", "random"}                                     | random      | The strategy used to choose the feature set at each node (only used if max_features != num_features). <br>Supported strategies are “best” to choose the best feature set and “random” to choose a random combination. <br>The algorithm generates 5 candidates at most to choose from in both strategies.                                                                                                                                            |
+
+\* Hyperparameter used by the support vector classifier of every node
+
+\*\* **Splitting in a STree node**
+
+The decision function is applied to the dataset and distances from samples to hyperplanes are computed in a matrix. This matrix has as many columns as classes the samples belongs to (if more than two, i.e. multiclass classification) or 1 column if it's a binary class dataset. In binary classification only one hyperplane is computed and therefore only one column is needed to store the distances of the samples to it. If three or more classes are present in the dataset we need as many hyperplanes as classes are there, and therefore one column per hyperplane is needed.
+
+In case of multiclass classification we have to decide which column take into account to make the split, that depends on hyperparameter _split_criteria_, if "impurity" is chosen then STree computes information gain of every split candidate using each column and chooses the one that maximize the information gain, otherwise STree choses the column with more samples with a predicted class (the column with more positive numbers in it).
+
+Once we have the column to take into account for the split, the algorithm splits samples with positive distances to hyperplane from the rest.
+
 ## Tests
 
-```python
-python -m unittest -v tests.Stree_test tests.Snode_test
+```bash
+python -m unittest -v stree.tests
 ```

codecov.yml

@@ -0,0 +1,12 @@
+overage:
+  status:
+    project:
+      default:
+        target: 90%
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: default
+  require_changes: false  
+  require_base: yes
+  require_head: yes       
+  branches: null

data/.gitignore

@@ -1,2 +0,0 @@
-*.csv
-*.txt

main.py

@@ -1,58 +1,29 @@
 import time
 from sklearn.model_selection import train_test_split
-from trees.Stree import Stree
+from sklearn.datasets import load_iris
+from stree import Stree
 
-random_state=1
+random_state = 1
 
-def load_creditcard(n_examples=0):
-    import pandas as pd
-    import numpy as np
-    import random
-    df = pd.read_csv('data/creditcard.csv')
-    print("Fraud: {0:.3f}% {1}".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))
-    print("Valid: {0:.3f}% {1}".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))
-    y = np.expand_dims(df.Class.values, axis=1)
-    X = df.drop(['Class', 'Time', 'Amount'], axis=1).values
-    if n_examples > 0:
-        # Take first n_examples samples
-        X = X[:n_examples, :]
-        y = y[:n_examples, :]
-    else:
-        # Take all the positive samples with a number of random negatives
-        if n_examples < 0:
-            Xt = X[(y == 1).ravel()]
-            yt = y[(y == 1).ravel()]
-            indices = random.sample(range(X.shape[0]), -1 * n_examples)
-            X = np.append(Xt, X[indices], axis=0)
-            y = np.append(yt, y[indices], axis=0)
-    print("X.shape", X.shape, " y.shape", y.shape)
-    print("Fraud: {0:.3f}% {1}".format(len(y[y == 1])*100/X.shape[0], len(y[y == 1])))
-    print("Valid: {0:.3f}% {1}".format(len(y[y == 0]) * 100 / X.shape[0], len(y[y == 0])))
-    Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7, shuffle=True, random_state=random_state, stratify=y)
-    return Xtrain, Xtest, ytrain, ytest
+X, y = load_iris(return_X_y=True)
 
-# data = load_creditcard(-5000) # Take all true samples + 5000 of the others
-# data = load_creditcard(5000)  # Take the first 5000 samples
-data = load_creditcard() # Take all the samples
-
-Xtrain = data[0]
-Xtest = data[1]
-ytrain = data[2]
-ytest = data[3]
+Xtrain, Xtest, ytrain, ytest = train_test_split(
+    X, y, test_size=0.3, random_state=random_state
+)
 
 now = time.time()
-clf = Stree(C=.01, random_state=random_state)
+print("Predicting with max_features=sqrt(n_features)")
+clf = Stree(C=0.01, random_state=random_state, max_features="auto")
+clf.fit(Xtrain, ytrain)
+print(f"Took {time.time() - now:.2f} seconds to train")
+print(clf)
+print(f"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}")
+print(f"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}")
+print("=" * 40)
+print("Predicting with max_features=n_features")
+clf = Stree(C=0.01, random_state=random_state)
 clf.fit(Xtrain, ytrain)
 print(f"Took {time.time() - now:.2f} seconds to train")
 print(clf)
 print(f"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}")
 print(f"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}")
-proba = clf.predict_proba(Xtest)
-print("Checking that we have correct probabilities, these are probabilities of sample belonging to class 1")
-res0 = proba[proba[:, 0] == 0]
-res1 = proba[proba[:, 0] == 0]
-print("++++++++++res0++++++++++++")
-print(res0[res0[:, 1] > .8])
-print("**********res1************")
-print(res1[res1[:, 1] < .4])
-print(clf.predict_proba(Xtest))

notebooks/benchmark.ipynb

@@ -0,0 +1,613 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Compare STree with different estimators"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup\n",
+    "Uncomment the next cell if STree is not already installed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#\n",
+    "# Google Colab setup\n",
+    "#\n",
+    "#!pip install git+https://github.com/doctorado-ml/stree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datetime, time\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.metrics import classification_report, confusion_matrix, f1_score\n",
+    "from sklearn.tree import DecisionTreeClassifier\n",
+    "from sklearn.naive_bayes import GaussianNB\n",
+    "from sklearn.neural_network import MLPClassifier\n",
+    "from sklearn.svm import LinearSVC\n",
+    "from stree import Stree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "if not os.path.isfile('data/creditcard.csv'):\n",
+    "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
+    "    !tar xzf creditcard.tgz"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2021-01-14 11:30:51\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(datetime.date.today(), time.strftime(\"%H:%M:%S\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load dataset and normalize values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Load Dataset\n",
+    "df = pd.read_csv('data/creditcard.csv')\n",
+    "df.shape\n",
+    "random_state = 2020"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fraud: 0.173% 492\n",
+      "Valid: 99.827% 284,315\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
+    "print(\"Valid: {0:.3f}% {1:,}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Normalize Amount\n",
+    "from sklearn.preprocessing import RobustScaler\n",
+    "values = RobustScaler().fit_transform(df.Amount.values.reshape(-1, 1))\n",
+    "df['Amount_Scaled'] = values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "X shape: (284807, 29)\n",
+      "y shape: (284807,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Remove unneeded features\n",
+    "y = df.Class.values\n",
+    "X = df.drop(['Class', 'Time', 'Amount'], axis=1).values\n",
+    "print(f\"X shape: {X.shape}\\ny shape: {y.shape}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build the models"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Divide dataset\n",
+    "train_size = .7\n",
+    "Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=train_size, shuffle=True, random_state=random_state, stratify=y)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Linear Tree\n",
+    "linear_tree = DecisionTreeClassifier(random_state=random_state)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Naive Bayes\n",
+    "naive_bayes = GaussianNB()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Stree\n",
+    "stree = Stree(random_state=random_state, C=.01, max_iter=1e3)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Neural Network\n",
+    "mlp = MLPClassifier(random_state=random_state, alpha=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# SVC (linear)\n",
+    "svc = LinearSVC(random_state=random_state, C=.01, max_iter=1e3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Do the test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def try_model(name, model):\n",
+    "    print(f\"************************** {name} **********************\")\n",
+    "    now = time.time()\n",
+    "    model.fit(Xtrain, ytrain)\n",
+    "    spent = time.time() - now\n",
+    "    print(f\"Train Model {name} took: {spent:.4} seconds\")\n",
+    "    predict = model.predict(Xtrain)\n",
+    "    predictt = model.predict(Xtest)\n",
+    "    print(f\"=========== {name} - Train {Xtrain.shape[0]:,} samples =============\",)\n",
+    "    print(classification_report(ytrain, predict, digits=6))\n",
+    "    print(f\"=========== {name} - Test {Xtest.shape[0]:,} samples =============\")\n",
+    "    print(classification_report(ytest, predictt, digits=6))\n",
+    "    print(\"Confusion Matrix in Train\")\n",
+    "    print(confusion_matrix(ytrain, predict))\n",
+    "    print(\"Confusion Matrix in Test\")\n",
+    "    print(confusion_matrix(ytest, predictt))\n",
+    "    return f1_score(ytest, predictt), spent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "************************** Linear Tree **********************\n",
+      "Train Model Linear Tree took: 10.25 seconds\n",
+      "=========== Linear Tree - Train 199,364 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   1.000000  1.000000  1.000000    199020\n",
+      "           1   1.000000  1.000000  1.000000       344\n",
+      "\n",
+      "    accuracy                       1.000000    199364\n",
+      "   macro avg   1.000000  1.000000  1.000000    199364\n",
+      "weighted avg   1.000000  1.000000  1.000000    199364\n",
+      "\n",
+      "=========== Linear Tree - Test 85,443 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999578  0.999613  0.999596     85295\n",
+      "           1   0.772414  0.756757  0.764505       148\n",
+      "\n",
+      "    accuracy                       0.999192     85443\n",
+      "   macro avg   0.885996  0.878185  0.882050     85443\n",
+      "weighted avg   0.999184  0.999192  0.999188     85443\n",
+      "\n",
+      "Confusion Matrix in Train\n",
+      "[[199020      0]\n",
+      " [     0    344]]\n",
+      "Confusion Matrix in Test\n",
+      "[[85262    33]\n",
+      " [   36   112]]\n",
+      "************************** Naive Bayes **********************\n",
+      "Train Model Naive Bayes took: 0.09943 seconds\n",
+      "=========== Naive Bayes - Train 199,364 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999692  0.978238  0.988849    199020\n",
+      "           1   0.061538  0.825581  0.114539       344\n",
+      "\n",
+      "    accuracy                       0.977975    199364\n",
+      "   macro avg   0.530615  0.901910  0.551694    199364\n",
+      "weighted avg   0.998073  0.977975  0.987340    199364\n",
+      "\n",
+      "=========== Naive Bayes - Test 85,443 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999712  0.977994  0.988734     85295\n",
+      "           1   0.061969  0.837838  0.115403       148\n",
+      "\n",
+      "    accuracy                       0.977751     85443\n",
+      "   macro avg   0.530841  0.907916  0.552068     85443\n",
+      "weighted avg   0.998088  0.977751  0.987221     85443\n",
+      "\n",
+      "Confusion Matrix in Train\n",
+      "[[194689   4331]\n",
+      " [    60    284]]\n",
+      "Confusion Matrix in Test\n",
+      "[[83418  1877]\n",
+      " [   24   124]]\n",
+      "************************** Stree (SVM Tree) **********************\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
+      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
+      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
+      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
+      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
+      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train Model Stree (SVM Tree) took: 28.47 seconds\n",
+      "=========== Stree (SVM Tree) - Train 199,364 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999623  0.999864  0.999744    199020\n",
+      "           1   0.908784  0.781977  0.840625       344\n",
+      "\n",
+      "    accuracy                       0.999488    199364\n",
+      "   macro avg   0.954204  0.890921  0.920184    199364\n",
+      "weighted avg   0.999467  0.999488  0.999469    199364\n",
+      "\n",
+      "=========== Stree (SVM Tree) - Test 85,443 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999637  0.999918  0.999777     85295\n",
+      "           1   0.943548  0.790541  0.860294       148\n",
+      "\n",
+      "    accuracy                       0.999555     85443\n",
+      "   macro avg   0.971593  0.895229  0.930036     85443\n",
+      "weighted avg   0.999540  0.999555  0.999536     85443\n",
+      "\n",
+      "Confusion Matrix in Train\n",
+      "[[198993     27]\n",
+      " [    75    269]]\n",
+      "Confusion Matrix in Test\n",
+      "[[85288     7]\n",
+      " [   31   117]]\n",
+      "************************** Neural Network **********************\n",
+      "Train Model Neural Network took: 9.76 seconds\n",
+      "=========== Neural Network - Train 199,364 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999247  0.999844  0.999545    199020\n",
+      "           1   0.862222  0.563953  0.681898       344\n",
+      "\n",
+      "    accuracy                       0.999092    199364\n",
+      "   macro avg   0.930734  0.781899  0.840722    199364\n",
+      "weighted avg   0.999010  0.999092  0.998997    199364\n",
+      "\n",
+      "=========== Neural Network - Test 85,443 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999356  0.999871  0.999613     85295\n",
+      "           1   0.894231  0.628378  0.738095       148\n",
+      "\n",
+      "    accuracy                       0.999228     85443\n",
+      "   macro avg   0.946793  0.814125  0.868854     85443\n",
+      "weighted avg   0.999173  0.999228  0.999160     85443\n",
+      "\n",
+      "Confusion Matrix in Train\n",
+      "[[198989     31]\n",
+      " [   150    194]]\n",
+      "Confusion Matrix in Test\n",
+      "[[85284    11]\n",
+      " [   55    93]]\n",
+      "************************** SVC (linear) **********************\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
+      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Train Model SVC (linear) took: 8.207 seconds\n",
+      "=========== SVC (linear) - Train 199,364 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999237  0.999859  0.999548    199020\n",
+      "           1   0.872727  0.558140  0.680851       344\n",
+      "\n",
+      "    accuracy                       0.999097    199364\n",
+      "   macro avg   0.935982  0.778999  0.840199    199364\n",
+      "weighted avg   0.999018  0.999097  0.998998    199364\n",
+      "\n",
+      "=========== SVC (linear) - Test 85,443 samples =============\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "           0   0.999344  0.999894  0.999619     85295\n",
+      "           1   0.910891  0.621622  0.738956       148\n",
+      "\n",
+      "    accuracy                       0.999239     85443\n",
+      "   macro avg   0.955117  0.810758  0.869287     85443\n",
+      "weighted avg   0.999191  0.999239  0.999168     85443\n",
+      "\n",
+      "Confusion Matrix in Train\n",
+      "[[198992     28]\n",
+      " [   152    192]]\n",
+      "Confusion Matrix in Test\n",
+      "[[85286     9]\n",
+      " [   56    92]]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Train & Test models\n",
+    "models = {\n",
+    "    'Linear Tree':linear_tree, 'Naive Bayes': naive_bayes, 'Stree (SVM Tree)': stree,  \n",
+    "    'Neural Network': mlp, 'SVC (linear)': svc\n",
+    "}\n",
+    "\n",
+    "best_f1 = 0\n",
+    "outcomes = []\n",
+    "for name, model in models.items():\n",
+    "    f1, time_spent = try_model(name, model)\n",
+    "    outcomes.append((name, f1, time_spent))\n",
+    "    if f1 > best_f1:\n",
+    "        best_model = name\n",
+    "        best_time = time_spent\n",
+    "        best_f1 = f1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "**************************************************************************************************************\n",
+      "*The best f1 model is Stree (SVM Tree), with a f1 score: 0.8603 in 28.4743 seconds with 0.7 samples in train dataset\n",
+      "**************************************************************************************************************\n",
+      "Model: Linear Tree\t Time:  10.25 seconds\t f1: 0.7645\n",
+      "Model: Naive Bayes\t Time:   0.10 seconds\t f1: 0.1154\n",
+      "Model: Stree (SVM Tree)\t Time:  28.47 seconds\t f1: 0.8603\n",
+      "Model: Neural Network\t Time:   9.76 seconds\t f1: 0.7381\n",
+      "Model: SVC (linear)\t Time:   8.21 seconds\t f1: 0.739\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"*\"*110)\n",
+    "print(f\"*The best f1 model is {best_model}, with a f1 score: {best_f1:.4} in {best_time:.6} seconds with {train_size:,} samples in train dataset\")\n",
+    "print(\"*\"*110)\n",
+    "for name, f1, time_spent in outcomes:\n",
+    "    print(f\"Model: {name}\\t Time: {time_spent:6.2f} seconds\\t f1: {f1:.4}\")"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "**************************************************************************************************************\n",
+    "*The best f1 model is Stree (SVM Tree), with a f1 score: 0.8603 in 28.4743 seconds with 0.7 samples in train dataset\n",
+    "**************************************************************************************************************\n",
+    "Model: Linear Tree\t Time:  10.25 seconds\t f1: 0.7645\n",
+    "Model: Naive Bayes\t Time:   0.10 seconds\t f1: 0.1154\n",
+    "Model: Stree (SVM Tree)\t Time:  28.47 seconds\t f1: 0.8603\n",
+    "Model: Neural Network\t Time:   9.76 seconds\t f1: 0.7381\n",
+    "Model: SVC (linear)\t Time:   8.21 seconds\t f1: 0.739"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'C': 0.01,\n",
+       " 'criterion': 'entropy',\n",
+       " 'degree': 3,\n",
+       " 'gamma': 'scale',\n",
+       " 'kernel': 'linear',\n",
+       " 'max_depth': None,\n",
+       " 'max_features': None,\n",
+       " 'max_iter': 1000.0,\n",
+       " 'min_samples_split': 0,\n",
+       " 'random_state': 2020,\n",
+       " 'split_criteria': 'impurity',\n",
+       " 'splitter': 'random',\n",
+       " 'tol': 0.0001}"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "stree.get_params()"
+   ]
+  }
+ ],
+ "metadata": {
+  "hide_input": false,
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.2"
+  },
+  "toc": {
+   "base_numbering": 1,
+   "nav_menu": {},
+   "number_sections": true,
+   "sideBar": true,
+   "skip_h1_title": false,
+   "title_cell": "Table of Contents",
+   "title_sidebar": "Contents",
+   "toc_cell": false,
+   "toc_position": {},
+   "toc_section_display": true,
+   "toc_window_display": false
+  },
+  "varInspector": {
+   "cols": {
+    "lenName": 16,
+    "lenType": 16,
+    "lenVar": 40
+   },
+   "kernels_config": {
+    "python": {
+     "delete_cmd_postfix": "",
+     "delete_cmd_prefix": "del ",
+     "library": "var_list.py",
+     "varRefreshCmd": "print(var_dic_list())"
+    },
+    "r": {
+     "delete_cmd_postfix": ") ",
+     "delete_cmd_prefix": "rm(",
+     "library": "var_list.r",
+     "varRefreshCmd": "cat(var_dic_list()) "
+    }
+   },
+   "position": {
+    "height": "392px",
+    "left": "1518px",
+    "right": "20px",
+    "top": "40px",
+    "width": "392px"
+   },
+   "types_to_exclude": [
+    "module",
+    "function",
+    "builtin_function_or_method",
+    "instance",
+    "_Feature"
+   ],
+   "window_display": true
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/ensemble.ipynb

@@ -0,0 +1,269 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Test Stree with AdaBoost and Bagging with different configurations"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup\n",
+    "Uncomment the next cell if STree is not already installed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#\n",
+    "# Google Colab setup\n",
+    "#\n",
+    "#!pip install git+https://github.com/doctorado-ml/stree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "import warnings\n",
+    "from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.exceptions import ConvergenceWarning\n",
+    "from stree import Stree\n",
+    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "if not os.path.isfile('data/creditcard.csv'):\n",
+    "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
+    "    !tar xzf creditcard.tgz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fraud: 0.173% 492\n",
+      "Valid: 99.827% 284315\n",
+      "X.shape (100492, 28)  y.shape (100492,)\n",
+      "Fraud: 0.651% 654\n",
+      "Valid: 99.349% 99838\n"
+     ]
+    }
+   ],
+   "source": [
+    "random_state=1\n",
+    "\n",
+    "def load_creditcard(n_examples=0):\n",
+    "    import pandas as pd\n",
+    "    import numpy as np\n",
+    "    import random\n",
+    "    df = pd.read_csv('data/creditcard.csv')\n",
+    "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
+    "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
+    "    y = df.Class\n",
+    "    X = df.drop(['Class', 'Time', 'Amount'], axis=1).values\n",
+    "    if n_examples > 0:\n",
+    "        # Take first n_examples samples\n",
+    "        X = X[:n_examples, :]\n",
+    "        y = y[:n_examples, :]\n",
+    "    else:\n",
+    "        # Take all the positive samples with a number of random negatives\n",
+    "        if n_examples < 0:\n",
+    "            Xt = X[(y == 1).ravel()]\n",
+    "            yt = y[(y == 1).ravel()]\n",
+    "            indices = random.sample(range(X.shape[0]), -1 * n_examples)\n",
+    "            X = np.append(Xt, X[indices], axis=0)\n",
+    "            y = np.append(yt, y[indices], axis=0)\n",
+    "    print(\"X.shape\", X.shape, \" y.shape\", y.shape)\n",
+    "    print(\"Fraud: {0:.3f}% {1}\".format(len(y[y == 1])*100/X.shape[0], len(y[y == 1])))\n",
+    "    print(\"Valid: {0:.3f}% {1}\".format(len(y[y == 0]) * 100 / X.shape[0], len(y[y == 0])))\n",
+    "    Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7, shuffle=True, random_state=random_state, stratify=y)\n",
+    "    return Xtrain, Xtest, ytrain, ytest\n",
+    "\n",
+    "# data = load_creditcard(-1000) # Take all true samples + 1000 of the others\n",
+    "# data = load_creditcard(5000)  # Take the first 5000 samples\n",
+    "# data = load_creditcard(0) # Take all the samples\n",
+    "data = load_creditcard(-100000)\n",
+    "\n",
+    "Xtrain = data[0]\n",
+    "Xtest = data[1]\n",
+    "ytrain = data[2]\n",
+    "ytest = data[3]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tests"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## STree alone with 100.000 samples and linear kernel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Score Train:  0.9984504719663368\n",
+      "Score Test:  0.9983415151917209\n",
+      "Took 26.09 seconds\n"
+     ]
+    }
+   ],
+   "source": [
+    "now = time.time()\n",
+    "clf = Stree(max_depth=3, random_state=random_state, max_iter=1e3)\n",
+    "clf.fit(Xtrain, ytrain)\n",
+    "print(\"Score Train: \", clf.score(Xtrain, ytrain))\n",
+    "print(\"Score Test: \", clf.score(Xtest, ytest))\n",
+    "print(f\"Took {time.time() - now:.2f} seconds\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Adaboost"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "n_estimators = 10\n",
+    "C = 7\n",
+    "max_depth = 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Kernel: linear\tTime: 43.49 seconds\tScore Train: 0.9980098\tScore Test: 0.9980762\n",
+      "Kernel: rbf\tTime: 8.86 seconds\tScore Train: 0.9934891\tScore Test: 0.9934987\n",
+      "Kernel: poly\tTime: 41.14 seconds\tScore Train: 0.9972279\tScore Test: 0.9973133\n"
+     ]
+    }
+   ],
+   "source": [
+    "for kernel in ['linear', 'rbf', 'poly']:\n",
+    "    now = time.time()\n",
+    "    clf = AdaBoostClassifier(base_estimator=Stree(C=C, kernel=kernel, max_depth=max_depth, random_state=random_state, max_iter=1e3), algorithm=\"SAMME\", n_estimators=n_estimators, random_state=random_state)\n",
+    "    clf.fit(Xtrain, ytrain)\n",
+    "    score_train = clf.score(Xtrain, ytrain)\n",
+    "    score_test = clf.score(Xtest, ytest)\n",
+    "    print(f\"Kernel: {kernel}\\tTime: {time.time() - now:.2f} seconds\\tScore Train: {score_train:.7f}\\tScore Test: {score_test:.7f}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Bagging"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "n_estimators = 10\n",
+    "C = 7\n",
+    "max_depth = 3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Kernel: linear\tTime: 187.51 seconds\tScore Train: 0.9984505\tScore Test: 0.9983083\n",
+      "Kernel: rbf\tTime: 73.65 seconds\tScore Train: 0.9993461\tScore Test: 0.9985074\n",
+      "Kernel: poly\tTime: 52.19 seconds\tScore Train: 0.9993461\tScore Test: 0.9987727\n"
+     ]
+    }
+   ],
+   "source": [
+    "for kernel in ['linear', 'rbf', 'poly']:\n",
+    "    now = time.time()\n",
+    "    clf = BaggingClassifier(base_estimator=Stree(C=C, kernel=kernel, max_depth=max_depth, random_state=random_state, max_iter=1e3), n_estimators=n_estimators, random_state=random_state)\n",
+    "    clf.fit(Xtrain, ytrain)\n",
+    "    score_train = clf.score(Xtrain, ytrain)\n",
+    "    score_test = clf.score(Xtest, ytest)\n",
+    "    print(f\"Kernel: {kernel}\\tTime: {time.time() - now:.2f} seconds\\tScore Train: {score_train:.7f}\\tScore Test: {score_test:.7f}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/features.ipynb

@@ -0,0 +1,560 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Test sample_weight, kernels, C, sklearn estimator"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup\n",
+    "Uncomment the next cell if STree is not already installed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#\n",
+    "# Google Colab setup\n",
+    "#\n",
+    "#!pip install git+https://github.com/doctorado-ml/stree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "import warnings\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from sklearn.svm import SVC\n",
+    "from sklearn.tree import DecisionTreeClassifier\n",
+    "from sklearn.utils.estimator_checks import check_estimator\n",
+    "from sklearn.datasets import make_classification, load_iris, load_wine\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.exceptions import ConvergenceWarning\n",
+    "from stree import Stree\n",
+    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "if not os.path.isfile('data/creditcard.csv'):\n",
+    "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
+    "    !tar xzf creditcard.tgz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fraud: 0.173% 492\n",
+      "Valid: 99.827% 284315\n",
+      "X.shape (5492, 28)  y.shape (5492,)\n",
+      "Fraud: 9.086% 499\n",
+      "Valid: 90.914% 4993\n",
+      "[0.09079084 0.09079084 0.09079084 0.09079084] [0.09101942 0.09101942 0.09101942 0.09101942]\n"
+     ]
+    }
+   ],
+   "source": [
+    "random_state=1\n",
+    "\n",
+    "def load_creditcard(n_examples=0):\n",
+    "    import pandas as pd\n",
+    "    import numpy as np\n",
+    "    import random\n",
+    "    df = pd.read_csv('data/creditcard.csv')\n",
+    "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
+    "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
+    "    y = df.Class\n",
+    "    X = df.drop(['Class', 'Time', 'Amount'], axis=1).values\n",
+    "    if n_examples > 0:\n",
+    "        # Take first n_examples samples\n",
+    "        X = X[:n_examples, :]\n",
+    "        y = y[:n_examples, :]\n",
+    "    else:\n",
+    "        # Take all the positive samples with a number of random negatives\n",
+    "        if n_examples < 0:\n",
+    "            Xt = X[(y == 1).ravel()]\n",
+    "            yt = y[(y == 1).ravel()]\n",
+    "            indices = random.sample(range(X.shape[0]), -1 * n_examples)\n",
+    "            X = np.append(Xt, X[indices], axis=0)\n",
+    "            y = np.append(yt, y[indices], axis=0)\n",
+    "    print(\"X.shape\", X.shape, \" y.shape\", y.shape)\n",
+    "    print(\"Fraud: {0:.3f}% {1}\".format(len(y[y == 1])*100/X.shape[0], len(y[y == 1])))\n",
+    "    print(\"Valid: {0:.3f}% {1}\".format(len(y[y == 0]) * 100 / X.shape[0], len(y[y == 0])))\n",
+    "    Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7, shuffle=True, random_state=random_state)\n",
+    "    return Xtrain, Xtest, ytrain, ytest\n",
+    "\n",
+    "data = load_creditcard(-5000) # Take all true samples with up to 5000 of the others\n",
+    "# data = load_creditcard(5000)  # Take the first 5000 samples\n",
+    "# data = load_creditcard(-1000) # Take 1000 samples\n",
+    "\n",
+    "Xtrain = data[0]\n",
+    "Xtest = data[1]\n",
+    "ytrain = data[2]\n",
+    "ytest = data[3]\n",
+    "_, data = np.unique(ytrain, return_counts=True)\n",
+    "wtrain = (data[1] / np.sum(data),  data[0] / np.sum(data))\n",
+    "_, data = np.unique(ytest, return_counts=True)\n",
+    "wtest = (data[1] / np.sum(data),  data[0] / np.sum(data))\n",
+    "# Set weights inverse to its count class in dataset\n",
+    "weights = np.ones(Xtrain.shape[0],)\n",
+    "weights[ytrain==0] = wtrain[0]\n",
+    "weights[ytrain==1] = wtrain[1]\n",
+    "weights_test = np.ones(Xtest.shape[0],)\n",
+    "weights_test[ytest==0] = wtest[0]\n",
+    "weights_test[ytest==1] = wtest[1]\n",
+    "print(weights[:4], weights_test[:4])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tests"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test sample_weights\n",
+    "Compute accuracy with weights in samples. The weights are set based on the inverse of the number of samples of each class"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Accuracy of Train without weights 0.9849115504682622\n",
+      "Accuracy of Train with    weights 0.9849115504682622\n",
+      "Accuracy of Tests without weights 0.9848300970873787\n",
+      "Accuracy of Tests with    weights 0.9805825242718447\n"
+     ]
+    }
+   ],
+   "source": [
+    "C = 23\n",
+    "print(\"Accuracy of Train without weights\", Stree(C=C, random_state=1).fit(Xtrain, ytrain).score(Xtrain, ytrain))\n",
+    "print(\"Accuracy of Train with    weights\", Stree(C=C, random_state=1).fit(Xtrain, ytrain, sample_weight=weights).score(Xtrain, ytrain))\n",
+    "print(\"Accuracy of Tests without weights\", Stree(C=C, random_state=1).fit(Xtrain, ytrain).score(Xtest, ytest))\n",
+    "print(\"Accuracy of Tests with    weights\", Stree(C=C, random_state=1).fit(Xtrain, ytrain, sample_weight=weights).score(Xtest, ytest))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test accuracy with different kernels\n",
+    "Compute accuracy on train and test set with default hyperparmeters of every kernel"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Time: 26.59s\tKernel: linear\tAccuracy_train: 0.9846514047866806\tAccuracy_test: 0.9848300970873787\n",
+      "Time: 0.56s\tKernel: rbf\tAccuracy_train: 0.9947970863683663\tAccuracy_test: 0.9866504854368932\n",
+      "Time: 0.23s\tKernel: poly\tAccuracy_train: 0.9955775234131113\tAccuracy_test: 0.9824029126213593\n"
+     ]
+    }
+   ],
+   "source": [
+    "random_state=1\n",
+    "for kernel in ['linear', 'rbf', 'poly']:\n",
+    "    now = time.time()\n",
+    "    clf = Stree(C=7, kernel=kernel, random_state=random_state).fit(Xtrain, ytrain)\n",
+    "    accuracy_train = clf.score(Xtrain, ytrain)\n",
+    "    accuracy_test = clf.score(Xtest, ytest)\n",
+    "    time_spent = time.time() - now\n",
+    "    print(f\"Time: {time_spent:.2f}s\\tKernel: {kernel}\\tAccuracy_train: {accuracy_train}\\tAccuracy_test: {accuracy_test}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test diferent values of C"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "************** C=0.001 ****************************\n",
+      "Classifier's accuracy (train): 0.9823\n",
+      "Classifier's accuracy (test) : 0.9836\n",
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down, <cgaf> - Leaf class=0 belief= 0.981455 impurity=0.1332 counts=(array([0, 1]), array([3493,   66]))\n",
+      "root - Up, <cgaf> - Leaf class=1 belief= 0.992982 impurity=0.0603 counts=(array([0, 1]), array([  2, 283]))\n",
+      "\n",
+      "**************************************************\n",
+      "************** C=0.01 ****************************\n",
+      "Classifier's accuracy (train): 0.9834\n",
+      "Classifier's accuracy (test) : 0.9842\n",
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down, <cgaf> - Leaf class=0 belief= 0.982288 impurity=0.1284 counts=(array([0, 1]), array([3494,   63]))\n",
+      "root - Up, <cgaf> - Leaf class=1 belief= 0.996516 impurity=0.0335 counts=(array([0, 1]), array([  1, 286]))\n",
+      "\n",
+      "**************************************************\n",
+      "************** C=1 ****************************\n",
+      "Classifier's accuracy (train): 0.9844\n",
+      "Classifier's accuracy (test) : 0.9848\n",
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
+      "root - Down - Down, <cgaf> - Leaf class=0 belief= 0.983108 impurity=0.1236 counts=(array([0, 1]), array([3492,   60]))\n",
+      "root - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
+      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
+      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n",
+      "\n",
+      "**************************************************\n",
+      "************** C=5 ****************************\n",
+      "Classifier's accuracy (train): 0.9847\n",
+      "Classifier's accuracy (test) : 0.9848\n",
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
+      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3492,   60]))\n",
+      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983385 impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
+      "root - Down - Down - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
+      "root - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
+      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
+      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n",
+      "\n",
+      "**************************************************\n",
+      "************** C=17 ****************************\n",
+      "Classifier's accuracy (train): 0.9847\n",
+      "Classifier's accuracy (test) : 0.9848\n",
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
+      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
+      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983380 impurity=0.1220 counts=(array([0, 1]), array([3491,   59]))\n",
+      "root - Down - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Down - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=1.0000 counts=(array([0, 1]), array([1, 1]))\n",
+      "root - Down - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Down - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
+      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
+      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
+      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n",
+      "\n",
+      "**************************************************\n",
+      "59.0161 secs\n"
+     ]
+    }
+   ],
+   "source": [
+    "t = time.time()\n",
+    "for C in (.001, .01, 1, 5, 17):\n",
+    "    clf = Stree(C=C, kernel=\"linear\", random_state=random_state)\n",
+    "    clf.fit(Xtrain, ytrain)\n",
+    "    print(f\"************** C={C} ****************************\")\n",
+    "    print(f\"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}\")\n",
+    "    print(f\"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}\")\n",
+    "    print(clf)\n",
+    "    print(f\"**************************************************\")\n",
+    "print(f\"{time.time() - t:.4f} secs\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test iterator\n",
+    "Check different ways of using the iterator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
+      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
+      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983380 impurity=0.1220 counts=(array([0, 1]), array([3491,   59]))\n",
+      "root - Down - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Down - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=1.0000 counts=(array([0, 1]), array([1, 1]))\n",
+      "root - Down - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Down - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
+      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
+      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
+      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n"
+     ]
+    }
+   ],
+   "source": [
+    "#check iterator\n",
+    "for i in list(clf):\n",
+    "    print(i)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
+      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
+      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
+      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983380 impurity=0.1220 counts=(array([0, 1]), array([3491,   59]))\n",
+      "root - Down - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Down - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=1.0000 counts=(array([0, 1]), array([1, 1]))\n",
+      "root - Down - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
+      "root - Down - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
+      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
+      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
+      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n"
+     ]
+    }
+   ],
+   "source": [
+    "#check iterator again\n",
+    "for i in clf:\n",
+    "    print(i)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test STree is a sklearn estimator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1 functools.partial(<function check_no_attributes_set_in_init at 0x16817f670>, 'Stree')\n",
+      "2 functools.partial(<function check_estimators_dtypes at 0x168179820>, 'Stree')\n",
+      "3 functools.partial(<function check_fit_score_takes_y at 0x168179700>, 'Stree')\n",
+      "4 functools.partial(<function check_sample_weights_pandas_series at 0x168174040>, 'Stree')\n",
+      "5 functools.partial(<function check_sample_weights_not_an_array at 0x168174160>, 'Stree')\n",
+      "6 functools.partial(<function check_sample_weights_list at 0x168174280>, 'Stree')\n",
+      "7 functools.partial(<function check_sample_weights_shape at 0x1681743a0>, 'Stree')\n",
+      "8 functools.partial(<function check_sample_weights_invariance at 0x1681744c0>, 'Stree', kind='ones')\n",
+      "10 functools.partial(<function check_estimators_fit_returns_self at 0x16817b8b0>, 'Stree')\n",
+      "11 functools.partial(<function check_estimators_fit_returns_self at 0x16817b8b0>, 'Stree', readonly_memmap=True)\n",
+      "12 functools.partial(<function check_complex_data at 0x168174670>, 'Stree')\n",
+      "13 functools.partial(<function check_dtype_object at 0x1681745e0>, 'Stree')\n",
+      "14 functools.partial(<function check_estimators_empty_data_messages at 0x1681799d0>, 'Stree')\n",
+      "15 functools.partial(<function check_pipeline_consistency at 0x1681795e0>, 'Stree')\n",
+      "16 functools.partial(<function check_estimators_nan_inf at 0x168179af0>, 'Stree')\n",
+      "17 functools.partial(<function check_estimators_overwrite_params at 0x16817f550>, 'Stree')\n",
+      "18 functools.partial(<function check_estimator_sparse_data at 0x168172ee0>, 'Stree')\n",
+      "19 functools.partial(<function check_estimators_pickle at 0x168179d30>, 'Stree')\n",
+      "20 functools.partial(<function check_estimator_get_tags_default_keys at 0x168181790>, 'Stree')\n",
+      "21 functools.partial(<function check_classifier_data_not_an_array at 0x16817f8b0>, 'Stree')\n",
+      "22 functools.partial(<function check_classifiers_one_label at 0x16817b430>, 'Stree')\n",
+      "23 functools.partial(<function check_classifiers_classes at 0x16817bd30>, 'Stree')\n",
+      "24 functools.partial(<function check_estimators_partial_fit_n_features at 0x168179e50>, 'Stree')\n",
+      "25 functools.partial(<function check_classifiers_train at 0x16817b550>, 'Stree')\n",
+      "26 functools.partial(<function check_classifiers_train at 0x16817b550>, 'Stree', readonly_memmap=True)\n",
+      "27 functools.partial(<function check_classifiers_train at 0x16817b550>, 'Stree', readonly_memmap=True, X_dtype='float32')\n",
+      "28 functools.partial(<function check_classifiers_regression_target at 0x168181280>, 'Stree')\n",
+      "29 functools.partial(<function check_supervised_y_no_nan at 0x1681720d0>, 'Stree')\n",
+      "30 functools.partial(<function check_supervised_y_2d at 0x16817baf0>, 'Stree')\n",
+      "31 functools.partial(<function check_estimators_unfitted at 0x16817b9d0>, 'Stree')\n",
+      "32 functools.partial(<function check_non_transformer_estimators_n_iter at 0x16817fdc0>, 'Stree')\n",
+      "33 functools.partial(<function check_decision_proba_consistency at 0x1681813a0>, 'Stree')\n",
+      "34 functools.partial(<function check_parameters_default_constructible at 0x16817fb80>, 'Stree')\n",
+      "35 functools.partial(<function check_methods_sample_order_invariance at 0x168174d30>, 'Stree')\n",
+      "36 functools.partial(<function check_methods_subset_invariance at 0x168174c10>, 'Stree')\n",
+      "37 functools.partial(<function check_fit2d_1sample at 0x168174e50>, 'Stree')\n",
+      "38 functools.partial(<function check_fit2d_1feature at 0x168174f70>, 'Stree')\n",
+      "39 functools.partial(<function check_get_params_invariance at 0x168181040>, 'Stree')\n",
+      "40 functools.partial(<function check_set_params at 0x168181160>, 'Stree')\n",
+      "41 functools.partial(<function check_dict_unchanged at 0x168174790>, 'Stree')\n",
+      "42 functools.partial(<function check_dont_overwrite_parameters at 0x168174940>, 'Stree')\n",
+      "43 functools.partial(<function check_fit_idempotent at 0x168181550>, 'Stree')\n",
+      "44 functools.partial(<function check_n_features_in at 0x1681815e0>, 'Stree')\n",
+      "45 functools.partial(<function check_fit1d at 0x1681790d0>, 'Stree')\n",
+      "46 functools.partial(<function check_fit2d_predict1d at 0x168174a60>, 'Stree')\n",
+      "47 functools.partial(<function check_requires_y_none at 0x168181670>, 'Stree')\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Make checks one by one\n",
+    "c = 0\n",
+    "checks = check_estimator(Stree(), generate_only=True)\n",
+    "for check in checks:\n",
+    "    c += 1\n",
+    "    if c == 9:\n",
+    "        pass\n",
+    "    else:\n",
+    "        print(c, check[1])\n",
+    "        check[1](check[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Check if the classifier is a sklearn estimator\n",
+    "check_estimator(Stree())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Compare to SVM"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "svc = SVC(C=7, kernel='rbf', gamma=.001, random_state=random_state)\n",
+    "clf = Stree(C=17, kernel='rbf', gamma=.001, random_state=random_state)\n",
+    "svc.fit(Xtrain, ytrain)\n",
+    "clf.fit(Xtrain, ytrain)\n",
+    "print(\"== Not Weighted ===\")\n",
+    "print(\"SVC train score ..:\", svc.score(Xtrain, ytrain))\n",
+    "print(\"STree train score :\", clf.score(Xtrain, ytrain))\n",
+    "print(\"SVC test score ...:\", svc.score(Xtest, ytest))\n",
+    "print(\"STree test score .:\", clf.score(Xtest, ytest))\n",
+    "svc.fit(Xtrain, ytrain, weights)\n",
+    "clf.fit(Xtrain, ytrain, weights)\n",
+    "print(\"==== Weighted =====\")\n",
+    "print(\"SVC train score ..:\", svc.score(Xtrain, ytrain))\n",
+    "print(\"STree train score :\", clf.score(Xtrain, ytrain))\n",
+    "print(\"SVC test score ...:\", svc.score(Xtest, ytest))\n",
+    "print(\"STree test score .:\", clf.score(Xtest, ytest))\n",
+    "print(\"*SVC test score ..:\", svc.score(Xtest, ytest, weights_test))\n",
+    "print(\"*STree test score :\", clf.score(Xtest, ytest, weights_test))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "print(clf)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test max_features"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for max_features in [None, \"auto\", \"log2\", 7, .5, .1, .7]:\n",
+    "    now = time.time()\n",
+    "    print(\"*\"*40)\n",
+    "    clf = Stree(random_state=random_state, max_features=max_features)\n",
+    "    clf.fit(Xtrain, ytrain)\n",
+    "    print(f\"max_features {max_features} = {clf.max_features_}\")\n",
+    "    print(\"Train score :\", clf.score(Xtrain, ytrain))\n",
+    "    print(\"Test score .:\", clf.score(Xtest, ytest))\n",
+    "    print(f\"Took {time.time() - now:.2f} seconds\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

notebooks/gridsearch.ipynb

@@ -0,0 +1,362 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Test Gridsearch\n",
+    "with different kernels and different configurations"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup\n",
+    "Uncomment the next cell if STree is not already installed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#\n",
+    "# Google Colab setup\n",
+    "#\n",
+    "#!pip install git+https://github.com/doctorado-ml/stree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "colab": {},
+    "colab_type": "code",
+    "id": "zIHKVxthDZEa"
+   },
+   "outputs": [],
+   "source": [
+    "from sklearn.ensemble import AdaBoostClassifier\n",
+    "from sklearn.svm import LinearSVC\n",
+    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
+    "from stree import Stree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "colab": {},
+    "colab_type": "code",
+    "id": "IEmq50QgDZEi"
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "if not os.path.isfile('data/creditcard.csv'):\n",
+    "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
+    "    !tar xzf creditcard.tgz"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "colab": {},
+    "colab_type": "code",
+    "id": "z9Q-YUfBDZEq",
+    "outputId": "afc822fb-f16a-4302-8a67-2b9e2880159b",
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fraud: 0.173% 492\n",
+      "Valid: 99.827% 284315\n",
+      "X.shape (1492, 28)  y.shape (1492,)\n",
+      "Fraud: 33.177% 495\n",
+      "Valid: 66.823% 997\n"
+     ]
+    }
+   ],
+   "source": [
+    "random_state=1\n",
+    "\n",
+    "def load_creditcard(n_examples=0):\n",
+    "    import pandas as pd\n",
+    "    import numpy as np\n",
+    "    import random\n",
+    "    df = pd.read_csv('data/creditcard.csv')\n",
+    "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
+    "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
+    "    y = df.Class\n",
+    "    X = df.drop(['Class', 'Time', 'Amount'], axis=1).values\n",
+    "    if n_examples > 0:\n",
+    "        # Take first n_examples samples\n",
+    "        X = X[:n_examples, :]\n",
+    "        y = y[:n_examples, :]\n",
+    "    else:\n",
+    "        # Take all the positive samples with a number of random negatives\n",
+    "        if n_examples < 0:\n",
+    "            Xt = X[(y == 1).ravel()]\n",
+    "            yt = y[(y == 1).ravel()]\n",
+    "            indices = random.sample(range(X.shape[0]), -1 * n_examples)\n",
+    "            X = np.append(Xt, X[indices], axis=0)\n",
+    "            y = np.append(yt, y[indices], axis=0)\n",
+    "    print(\"X.shape\", X.shape, \" y.shape\", y.shape)\n",
+    "    print(\"Fraud: {0:.3f}% {1}\".format(len(y[y == 1])*100/X.shape[0], len(y[y == 1])))\n",
+    "    print(\"Valid: {0:.3f}% {1}\".format(len(y[y == 0]) * 100 / X.shape[0], len(y[y == 0])))\n",
+    "    Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7, shuffle=True, random_state=random_state, stratify=y)\n",
+    "    return Xtrain, Xtest, ytrain, ytest\n",
+    "\n",
+    "data = load_creditcard(-1000) # Take all true samples + 1000 of the others\n",
+    "# data = load_creditcard(5000)  # Take the first 5000 samples\n",
+    "# data = load_creditcard(0) # Take all the samples\n",
+    "\n",
+    "Xtrain = data[0]\n",
+    "Xtest = data[1]\n",
+    "ytrain = data[2]\n",
+    "ytest = data[3]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "colab": {},
+    "colab_type": "code",
+    "id": "HmX3kR4PDZEw"
+   },
+   "outputs": [],
+   "source": [
+    "parameters = [{\n",
+    "    'base_estimator': [Stree(random_state=random_state)],\n",
+    "    'n_estimators': [10, 25],\n",
+    "    'learning_rate': [.5, 1],\n",
+    "    'base_estimator__split_criteria': ['max_samples', 'impurity'],\n",
+    "    'base_estimator__tol': [.1,  1e-02],\n",
+    "    'base_estimator__max_depth': [3, 5, 7],\n",
+    "    'base_estimator__C': [1, 7, 55],\n",
+    "    'base_estimator__kernel': ['linear']\n",
+    "},\n",
+    "{\n",
+    "    'base_estimator': [Stree(random_state=random_state)],\n",
+    "    'n_estimators': [10, 25],\n",
+    "    'learning_rate': [.5, 1],\n",
+    "    'base_estimator__split_criteria': ['max_samples', 'impurity'],\n",
+    "    'base_estimator__tol': [.1,  1e-02],\n",
+    "    'base_estimator__max_depth': [3, 5, 7],\n",
+    "    'base_estimator__C': [1, 7, 55],\n",
+    "    'base_estimator__degree': [3, 5, 7],\n",
+    "    'base_estimator__kernel': ['poly']\n",
+    "},\n",
+    "{\n",
+    "    'base_estimator': [Stree(random_state=random_state)],\n",
+    "    'n_estimators': [10, 25],\n",
+    "    'learning_rate': [.5, 1],\n",
+    "    'base_estimator__split_criteria': ['max_samples', 'impurity'],\n",
+    "    'base_estimator__tol': [.1,  1e-02],\n",
+    "    'base_estimator__max_depth': [3, 5, 7],\n",
+    "    'base_estimator__C': [1, 7, 55],\n",
+    "    'base_estimator__gamma': [.1, 1, 10],\n",
+    "    'base_estimator__kernel': ['rbf']\n",
+    "}]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'C': 1.0,\n",
+       " 'criterion': 'entropy',\n",
+       " 'degree': 3,\n",
+       " 'gamma': 'scale',\n",
+       " 'kernel': 'linear',\n",
+       " 'max_depth': None,\n",
+       " 'max_features': None,\n",
+       " 'max_iter': 100000.0,\n",
+       " 'min_samples_split': 0,\n",
+       " 'random_state': None,\n",
+       " 'split_criteria': 'impurity',\n",
+       " 'splitter': 'random',\n",
+       " 'tol': 0.0001}"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "Stree().get_params()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "colab": {},
+    "colab_type": "code",
+    "id": "CrcB8o6EDZE5",
+    "outputId": "7703413a-d563-4289-a13b-532f38f82762",
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fitting 5 folds for each of 1008 candidates, totalling 5040 fits\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
+      "[Parallel(n_jobs=-1)]: Done  40 tasks      | elapsed:    1.6s\n",
+      "[Parallel(n_jobs=-1)]: Done 130 tasks      | elapsed:    3.1s\n",
+      "[Parallel(n_jobs=-1)]: Done 256 tasks      | elapsed:    5.5s\n",
+      "[Parallel(n_jobs=-1)]: Done 418 tasks      | elapsed:    9.3s\n",
+      "[Parallel(n_jobs=-1)]: Done 616 tasks      | elapsed:   18.6s\n",
+      "[Parallel(n_jobs=-1)]: Done 850 tasks      | elapsed:   28.2s\n",
+      "[Parallel(n_jobs=-1)]: Done 1120 tasks      | elapsed:   35.4s\n",
+      "[Parallel(n_jobs=-1)]: Done 1426 tasks      | elapsed:   43.5s\n",
+      "[Parallel(n_jobs=-1)]: Done 1768 tasks      | elapsed:   51.3s\n",
+      "[Parallel(n_jobs=-1)]: Done 2146 tasks      | elapsed:  1.0min\n",
+      "[Parallel(n_jobs=-1)]: Done 2560 tasks      | elapsed:  1.2min\n",
+      "[Parallel(n_jobs=-1)]: Done 3010 tasks      | elapsed:  1.4min\n",
+      "[Parallel(n_jobs=-1)]: Done 3496 tasks      | elapsed:  1.7min\n",
+      "[Parallel(n_jobs=-1)]: Done 4018 tasks      | elapsed:  2.1min\n",
+      "[Parallel(n_jobs=-1)]: Done 4576 tasks      | elapsed:  2.6min\n",
+      "[Parallel(n_jobs=-1)]: Done 5040 out of 5040 | elapsed:  2.9min finished\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "GridSearchCV(estimator=AdaBoostClassifier(algorithm='SAMME', random_state=1),\n",
+       "             n_jobs=-1,\n",
+       "             param_grid=[{'base_estimator': [Stree(C=55, max_depth=7,\n",
+       "                                                   random_state=1,\n",
+       "                                                   split_criteria='max_samples',\n",
+       "                                                   tol=0.1)],\n",
+       "                          'base_estimator__C': [1, 7, 55],\n",
+       "                          'base_estimator__kernel': ['linear'],\n",
+       "                          'base_estimator__max_depth': [3, 5, 7],\n",
+       "                          'base_estimator__split_criteria': ['max_samples',\n",
+       "                                                             'impuri...\n",
+       "                         {'base_estimator': [Stree(random_state=1)],\n",
+       "                          'base_estimator__C': [1, 7, 55],\n",
+       "                          'base_estimator__gamma': [0.1, 1, 10],\n",
+       "                          'base_estimator__kernel': ['rbf'],\n",
+       "                          'base_estimator__max_depth': [3, 5, 7],\n",
+       "                          'base_estimator__split_criteria': ['max_samples',\n",
+       "                                                             'impurity'],\n",
+       "                          'base_estimator__tol': [0.1, 0.01],\n",
+       "                          'learning_rate': [0.5, 1],\n",
+       "                          'n_estimators': [10, 25]}],\n",
+       "             return_train_score=True, verbose=5)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "clf = AdaBoostClassifier(random_state=random_state, algorithm=\"SAMME\")\n",
+    "grid = GridSearchCV(clf, parameters, verbose=5, n_jobs=-1, return_train_score=True)\n",
+    "grid.fit(Xtrain, ytrain)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "colab": {},
+    "colab_type": "code",
+    "id": "ZjX88NoYDZE8",
+    "outputId": "285163c8-fa33-4915-8ae7-61c4f7844344",
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Best estimator:  AdaBoostClassifier(algorithm='SAMME',\n",
+      "                   base_estimator=Stree(C=55, max_depth=7, random_state=1,\n",
+      "                                        split_criteria='max_samples', tol=0.1),\n",
+      "                   learning_rate=0.5, n_estimators=25, random_state=1)\n",
+      "Best hyperparameters:  {'base_estimator': Stree(C=55, max_depth=7, random_state=1, split_criteria='max_samples', tol=0.1), 'base_estimator__C': 55, 'base_estimator__kernel': 'linear', 'base_estimator__max_depth': 7, 'base_estimator__split_criteria': 'max_samples', 'base_estimator__tol': 0.1, 'learning_rate': 0.5, 'n_estimators': 25}\n",
+      "Best accuracy:  0.9511777695988222\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Best estimator: \", grid.best_estimator_)\n",
+    "print(\"Best hyperparameters: \", grid.best_params_)\n",
+    "print(\"Best accuracy: \", grid.best_score_)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Best estimator:  AdaBoostClassifier(algorithm='SAMME',\n",
+    "                   base_estimator=Stree(C=55, max_depth=7, random_state=1,\n",
+    "                                        split_criteria='max_samples', tol=0.1),\n",
+    "                   learning_rate=0.5, n_estimators=25, random_state=1)\n",
+    "Best hyperparameters:  {'base_estimator': Stree(C=55, max_depth=7, random_state=1, split_criteria='max_samples', tol=0.1), 'base_estimator__C': 55, 'base_estimator__kernel': 'linear', 'base_estimator__max_depth': 7, 'base_estimator__split_criteria': 'max_samples', 'base_estimator__tol': 0.1, 'learning_rate': 0.5, 'n_estimators': 25}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Best accuracy:  0.9511777695988222"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "name": "gridsearch.ipynb",
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

pyproject.toml

@@ -0,0 +1,16 @@
+[tool.black]
+line-length = 79
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''

requirements.txt

@@ -1,3 +1,4 @@
-numpy==1.18.2
-scikit-learn==0.22.2
-pandas==1.0.3
+numpy
+scikit-learn
+pandas
+ipympl

runtime.txt

@@ -0,0 +1 @@
+python-3.8

setup.py

@@ -0,0 +1,36 @@
+import setuptools
+
+__version__ = "1.0rc1"
+__author__ = "Ricardo Montañana Gómez"
+
+
+def readme():
+    with open("README.md") as f:
+        return f.read()
+
+
+setuptools.setup(
+    name="STree",
+    version=__version__,
+    license="MIT License",
+    description="Oblique decision tree with svm nodes",
+    long_description=readme(),
+    long_description_content_type="text/markdown",
+    packages=setuptools.find_packages(),
+    url="https://github.com/doctorado-ml/stree",
+    author=__author__,
+    author_email="ricardo.montanana@alu.uclm.es",
+    keywords="scikit-learn oblique-classifier oblique-decision-tree decision-\
+    tree svm svc",
+    classifiers=[
+        "Development Status :: 4 - Beta",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3.8",
+        "Natural Language :: English",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "Intended Audience :: Science/Research",
+    ],
+    install_requires=["scikit-learn", "numpy", "ipympl"],
+    test_suite="stree.tests",
+    zip_safe=False,
+)

stree/Strees.py

@@ -0,0 +1,872 @@
+"""
+__author__ = "Ricardo Montañana Gómez"
+__copyright__ = "Copyright 2020, Ricardo Montañana Gómez"
+__license__ = "MIT"
+__version__ = "0.9"
+Build an oblique tree classifier based on SVM nodes
+"""
+
+import os
+import numbers
+import random
+import warnings
+from math import log, factorial
+from typing import Optional
+import numpy as np
+from sklearn.base import BaseEstimator, ClassifierMixin
+from sklearn.svm import SVC, LinearSVC
+from sklearn.utils import check_consistent_length
+from sklearn.utils.multiclass import check_classification_targets
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.utils.validation import (
+    check_X_y,
+    check_array,
+    check_is_fitted,
+    _check_sample_weight,
+)
+from sklearn.metrics._classification import _weighted_sum, _check_targets
+
+
+class Snode:
+    """Nodes of the tree that keeps the svm classifier and if testing the
+    dataset assigned to it
+    """
+
+    def __init__(
+        self,
+        clf: SVC,
+        X: np.ndarray,
+        y: np.ndarray,
+        features: np.array,
+        impurity: float,
+        title: str,
+        weight: np.ndarray = None,
+    ):
+        self._clf = clf
+        self._title = title
+        self._belief = 0.0
+        # Only store dataset in Testing
+        self._X = X if os.environ.get("TESTING", "NS") != "NS" else None
+        self._y = y
+        self._down = None
+        self._up = None
+        self._class = None
+        self._feature = None
+        self._sample_weight = (
+            weight if os.environ.get("TESTING", "NS") != "NS" else None
+        )
+        self._features = features
+        self._impurity = impurity
+        self._partition_column: int = -1
+
+    @classmethod
+    def copy(cls, node: "Snode") -> "Snode":
+        return cls(
+            node._clf,
+            node._X,
+            node._y,
+            node._features,
+            node._impurity,
+            node._title,
+        )
+
+    def set_partition_column(self, col: int):
+        self._partition_column = col
+
+    def get_partition_column(self) -> int:
+        return self._partition_column
+
+    def set_down(self, son):
+        self._down = son
+
+    def set_up(self, son):
+        self._up = son
+
+    def is_leaf(self) -> bool:
+        return self._up is None and self._down is None
+
+    def get_down(self) -> "Snode":
+        return self._down
+
+    def get_up(self) -> "Snode":
+        return self._up
+
+    def make_predictor(self):
+        """Compute the class of the predictor and its belief based on the
+        subdataset of the node only if it is a leaf
+        """
+        if not self.is_leaf():
+            return
+        classes, card = np.unique(self._y, return_counts=True)
+        if len(classes) > 1:
+            max_card = max(card)
+            self._class = classes[card == max_card][0]
+            self._belief = max_card / np.sum(card)
+        else:
+            self._belief = 1
+            try:
+                self._class = classes[0]
+            except IndexError:
+                self._class = None
+
+    def __str__(self) -> str:
+        count_values = np.unique(self._y, return_counts=True)
+        if self.is_leaf():
+            return (
+                f"{self._title} - Leaf class={self._class} belief="
+                f"{self._belief: .6f} impurity={self._impurity:.4f} "
+                f"counts={count_values}"
+            )
+        else:
+            return (
+                f"{self._title} feaures={self._features} impurity="
+                f"{self._impurity:.4f} "
+                f"counts={count_values}"
+            )
+
+
+class Siterator:
+    """Stree preorder iterator"""
+
+    def __init__(self, tree: Snode):
+        self._stack = []
+        self._push(tree)
+
+    def _push(self, node: Snode):
+        if node is not None:
+            self._stack.append(node)
+
+    def __next__(self) -> Snode:
+        if len(self._stack) == 0:
+            raise StopIteration()
+        node = self._stack.pop()
+        self._push(node.get_up())
+        self._push(node.get_down())
+        return node
+
+
+class Splitter:
+    def __init__(
+        self,
+        clf: SVC = None,
+        criterion: str = None,
+        splitter_type: str = None,
+        criteria: str = None,
+        min_samples_split: int = None,
+        random_state=None,
+    ):
+        self._clf = clf
+        self._random_state = random_state
+        if random_state is not None:
+            random.seed(random_state)
+        self._criterion = criterion
+        self._min_samples_split = min_samples_split
+        self._criteria = criteria
+        self._splitter_type = splitter_type
+
+        if clf is None:
+            raise ValueError(f"clf has to be a sklearn estimator, got({clf})")
+
+        if criterion not in ["gini", "entropy"]:
+            raise ValueError(
+                f"criterion must be gini or entropy got({criterion})"
+            )
+
+        if criteria not in [
+            "max_samples",
+            "impurity",
+        ]:
+            raise ValueError(
+                f"criteria has to be max_samples or impurity; got ({criteria})"
+            )
+
+        if splitter_type not in ["random", "best"]:
+            raise ValueError(
+                f"splitter must be either random or best, got({splitter_type})"
+            )
+        self.criterion_function = getattr(self, f"_{self._criterion}")
+        self.decision_criteria = getattr(self, f"_{self._criteria}")
+
+    def partition_impurity(self, y: np.array) -> np.array:
+        return self.criterion_function(y)
+
+    @staticmethod
+    def _gini(y: np.array) -> float:
+        _, count = np.unique(y, return_counts=True)
+        return 1 - np.sum(np.square(count / np.sum(count)))
+
+    @staticmethod
+    def _entropy(y: np.array) -> float:
+        """Compute entropy of a labels set
+
+        Parameters
+        ----------
+        y : np.array
+            set of labels
+
+        Returns
+        -------
+        float
+            entropy
+        """
+        n_labels = len(y)
+        if n_labels <= 1:
+            return 0
+        counts = np.bincount(y)
+        proportions = counts / n_labels
+        n_classes = np.count_nonzero(proportions)
+        if n_classes <= 1:
+            return 0
+        entropy = 0.0
+        # Compute standard entropy.
+        for prop in proportions:
+            if prop != 0.0:
+                entropy -= prop * log(prop, n_classes)
+        return entropy
+
+    def information_gain(
+        self, labels: np.array, labels_up: np.array, labels_dn: np.array
+    ) -> float:
+        """Compute information gain of a split candidate
+
+        Parameters
+        ----------
+        labels : np.array
+            labels of the dataset
+        labels_up : np.array
+            labels of one side
+        labels_dn : np.array
+            labels on the other side
+
+        Returns
+        -------
+        float
+            information gain
+        """
+        imp_prev = self.criterion_function(labels)
+        card_up = card_dn = imp_up = imp_dn = 0
+        if labels_up is not None:
+            card_up = labels_up.shape[0]
+            imp_up = self.criterion_function(labels_up)
+        if labels_dn is not None:
+            card_dn = labels_dn.shape[0] if labels_dn is not None else 0
+            imp_dn = self.criterion_function(labels_dn)
+        samples = card_up + card_dn
+        if samples == 0:
+            return 0.0
+        else:
+            result = (
+                imp_prev
+                - (card_up / samples) * imp_up
+                - (card_dn / samples) * imp_dn
+            )
+            return result
+
+    def _select_best_set(
+        self, dataset: np.array, labels: np.array, features_sets: list
+    ) -> list:
+        max_gain = 0
+        selected = None
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        for feature_set in features_sets:
+            self._clf.fit(dataset[:, feature_set], labels)
+            node = Snode(
+                self._clf, dataset, labels, feature_set, 0.0, "subset"
+            )
+            self.partition(dataset, node, train=True)
+            y1, y2 = self.part(labels)
+            gain = self.information_gain(labels, y1, y2)
+            if gain > max_gain:
+                max_gain = gain
+                selected = feature_set
+        return selected if selected is not None else feature_set
+
+    @staticmethod
+    def _generate_spaces(features: int, max_features: int) -> list:
+        """Generate at most 5 feature random combinations
+
+        Parameters
+        ----------
+        features : int
+            number of features in each combination
+        max_features : int
+            number of features in dataset
+
+        Returns
+        -------
+        list
+            list with up to 5 combination of features randomly selected
+        """
+        comb = set()
+        # Generate at most 5 combinations
+        if max_features == features:
+            set_length = 1
+        else:
+            number = factorial(features) / (
+                factorial(max_features) * factorial(features - max_features)
+            )
+            set_length = min(5, number)
+        while len(comb) < set_length:
+            comb.add(
+                tuple(sorted(random.sample(range(features), max_features)))
+            )
+        return list(comb)
+
+    def _get_subspaces_set(
+        self, dataset: np.array, labels: np.array, max_features: int
+    ) -> np.array:
+        """Compute the indices of the features selected by splitter depending
+        on the self._splitter_type hyper parameter
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (<= number of features in dataset)
+
+        Returns
+        -------
+        np.array
+            indices of the features selected
+        """
+        features_sets = self._generate_spaces(dataset.shape[1], max_features)
+        if len(features_sets) > 1:
+            if self._splitter_type == "random":
+                index = random.randint(0, len(features_sets) - 1)
+                return features_sets[index]
+            else:
+                return self._select_best_set(dataset, labels, features_sets)
+        else:
+            return features_sets[0]
+
+    def get_subspace(
+        self, dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Return a subspace of the selected dataset of max_features length.
+        Depending on hyperparmeter
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples (# samples, # features)
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features to form the subspace
+
+        Returns
+        -------
+        tuple
+            tuple with the dataset with only the features selected  and the
+            indices of the features selected
+        """
+        indices = self._get_subspaces_set(dataset, labels, max_features)
+        return dataset[:, indices], indices
+
+    def _impurity(self, data: np.array, y: np.array) -> np.array:
+        """return column of dataset to be taken into account to split dataset
+
+        Parameters
+        ----------
+        data : np.array
+            distances to hyper plane of every class
+        y : np.array
+            vector of labels (classes)
+
+        Returns
+        -------
+        np.array
+            column of dataset to be taken into account to split dataset
+        """
+        max_gain = 0
+        selected = -1
+        for col in range(data.shape[1]):
+            tup = y[data[:, col] > 0]
+            tdn = y[data[:, col] <= 0]
+            info_gain = self.information_gain(y, tup, tdn)
+            if info_gain > max_gain:
+                selected = col
+                max_gain = info_gain
+        return selected
+
+    @staticmethod
+    def _max_samples(data: np.array, y: np.array) -> np.array:
+        """return column of dataset to be taken into account to split dataset
+
+        Parameters
+        ----------
+        data : np.array
+            distances to hyper plane of every class
+        y : np.array
+            column of dataset to be taken into account to split dataset
+
+        Returns
+        -------
+        np.array
+            column of dataset to be taken into account to split dataset
+        """
+        # select the class with max number of samples
+        _, samples = np.unique(y, return_counts=True)
+        return np.argmax(samples)
+
+    def partition(self, samples: np.array, node: Snode, train: bool):
+        """Set the criteria to split arrays. Compute the indices of the samples
+        that should go to one side of the tree (up)
+        """
+        # data contains the distances of every sample to every class hyperplane
+        # array of (m, nc) nc = # classes
+        data = self._distances(node, samples)
+        if data.shape[0] < self._min_samples_split:
+            # there aren't enough samples to split
+            self._up = np.ones((data.shape[0]), dtype=bool)
+            return
+        if data.ndim > 1:
+            # split criteria for multiclass
+            # Convert data to a (m, 1) array selecting values for samples
+            if train:
+                # in train time we have to compute the column to take into
+                # account to split the dataset
+                col = self.decision_criteria(data, node._y)
+                node.set_partition_column(col)
+            else:
+                # in predcit time just use the column computed in train time
+                # is taking the classifier of class <col>
+                col = node.get_partition_column()
+                if col == -1:
+                    # No partition is producing information gain
+                    data = np.ones(data.shape)
+            data = data[:, col]
+        self._up = data > 0
+
+    def part(self, origin: np.array) -> list:
+        """Split an array in two based on indices (self._up) and its complement
+        partition has to be called first to establish up indices
+
+        Parameters
+        ----------
+        origin : np.array
+            dataset to split
+
+        Returns
+        -------
+        list
+            list with two splits of the array
+        """
+        down = ~self._up
+        return [
+            origin[self._up] if any(self._up) else None,
+            origin[down] if any(down) else None,
+        ]
+
+    @staticmethod
+    def _distances(node: Snode, data: np.ndarray) -> np.array:
+        """Compute distances of the samples to the hyperplane of the node
+
+        Parameters
+        ----------
+        node : Snode
+            node containing the svm classifier
+        data : np.ndarray
+            samples to compute distance to hyperplane
+
+        Returns
+        -------
+        np.array
+            array of shape (m, nc) with the distances of every sample to
+            the hyperplane of every class. nc = # of classes
+        """
+        return node._clf.decision_function(data[:, node._features])
+
+
+class Stree(BaseEstimator, ClassifierMixin):
+    """Estimator that is based on binary trees of svm nodes
+    can deal with sample_weights in predict, used in boosting sklearn methods
+    inheriting from BaseEstimator implements get_params and set_params methods
+    inheriting from ClassifierMixin implement the attribute _estimator_type
+    with "classifier" as value
+    """
+
+    def __init__(
+        self,
+        C: float = 1.0,
+        kernel: str = "linear",
+        max_iter: int = 1e5,
+        random_state: int = None,
+        max_depth: int = None,
+        tol: float = 1e-4,
+        degree: int = 3,
+        gamma="scale",
+        split_criteria: str = "impurity",
+        criterion: str = "entropy",
+        min_samples_split: int = 0,
+        max_features=None,
+        splitter: str = "random",
+    ):
+        self.max_iter = max_iter
+        self.C = C
+        self.kernel = kernel
+        self.random_state = random_state
+        self.max_depth = max_depth
+        self.tol = tol
+        self.gamma = gamma
+        self.degree = degree
+        self.min_samples_split = min_samples_split
+        self.split_criteria = split_criteria
+        self.max_features = max_features
+        self.criterion = criterion
+        self.splitter = splitter
+
+    def _more_tags(self) -> dict:
+        """Required by sklearn to supply features of the classifier
+        make mandatory the labels array
+
+        :return: the tag required
+        :rtype: dict
+        """
+        return {"requires_y": True}
+
+    def fit(
+        self, X: np.ndarray, y: np.ndarray, sample_weight: np.array = None
+    ) -> "Stree":
+        """Build the tree based on the dataset of samples and its labels
+
+        Returns
+        -------
+        Stree
+            itself to be able to chain actions: fit().predict() ...
+
+        Raises
+        ------
+        ValueError
+            if C < 0
+        ValueError
+            if max_depth < 1
+        ValueError
+            if all samples have 0 or negative weights
+        """
+        # Check parameters are Ok.
+        if self.C < 0:
+            raise ValueError(
+                f"Penalty term must be positive... got (C={self.C:f})"
+            )
+        self.__max_depth = (
+            np.iinfo(np.int32).max
+            if self.max_depth is None
+            else self.max_depth
+        )
+        if self.__max_depth < 1:
+            raise ValueError(
+                f"Maximum depth has to be greater than 1... got (max_depth=\
+                    {self.max_depth})"
+            )
+
+        check_classification_targets(y)
+        X, y = check_X_y(X, y)
+        sample_weight = _check_sample_weight(
+            sample_weight, X, dtype=np.float64
+        )
+        if not any(sample_weight):
+            raise ValueError(
+                "Invalid input - all samples have zero or negative weights."
+            )
+        check_classification_targets(y)
+        # Initialize computed parameters
+        self.splitter_ = Splitter(
+            clf=self._build_clf(),
+            criterion=self.criterion,
+            splitter_type=self.splitter,
+            criteria=self.split_criteria,
+            random_state=self.random_state,
+            min_samples_split=self.min_samples_split,
+        )
+        if self.random_state is not None:
+            random.seed(self.random_state)
+        self.classes_, y = np.unique(y, return_inverse=True)
+        self.n_classes_ = self.classes_.shape[0]
+        self.n_iter_ = self.max_iter
+        self.depth_ = 0
+        self.n_features_ = X.shape[1]
+        self.n_features_in_ = X.shape[1]
+        self.max_features_ = self._initialize_max_features()
+        self.tree_ = self.train(X, y, sample_weight, 1, "root")
+        self._build_predictor()
+        self.X_ = X
+        self.y_ = y
+        return self
+
+    def train(
+        self,
+        X: np.ndarray,
+        y: np.ndarray,
+        sample_weight: np.ndarray,
+        depth: int,
+        title: str,
+    ) -> Optional[Snode]:
+        """Recursive function to split the original dataset into predictor
+        nodes (leaves)
+
+        Parameters
+        ----------
+        X : np.ndarray
+            samples dataset
+        y : np.ndarray
+            samples labels
+        sample_weight : np.ndarray
+            weight of samples. Rescale C per sample.
+        depth : int
+            actual depth in the tree
+        title : str
+            description of the node
+
+        Returns
+        -------
+        Optional[Snode]
+            binary tree
+        """
+        if depth > self.__max_depth:
+            return None
+        # Mask samples with 0 weight
+        if any(sample_weight == 0):
+            indices_zero = sample_weight == 0
+            X = X[~indices_zero, :]
+            y = y[~indices_zero]
+            sample_weight = sample_weight[~indices_zero]
+        if np.unique(y).shape[0] == 1:
+            # only 1 class => pure dataset
+            return Snode(
+                clf=None,
+                X=X,
+                y=y,
+                features=X.shape[1],
+                impurity=0.0,
+                title=title + ", <pure>",
+                weight=sample_weight,
+            )
+        # Train the model
+        clf = self._build_clf()
+        Xs, features = self.splitter_.get_subspace(X, y, self.max_features_)
+        clf.fit(Xs, y, sample_weight=sample_weight)
+        impurity = self.splitter_.partition_impurity(y)
+        node = Snode(clf, X, y, features, impurity, title, sample_weight)
+        self.depth_ = max(depth, self.depth_)
+        self.splitter_.partition(X, node, True)
+        X_U, X_D = self.splitter_.part(X)
+        y_u, y_d = self.splitter_.part(y)
+        sw_u, sw_d = self.splitter_.part(sample_weight)
+        if X_U is None or X_D is None:
+            # didn't part anything
+            return Snode(
+                clf,
+                X,
+                y,
+                features=X.shape[1],
+                impurity=impurity,
+                title=title + ", <cgaf>",
+                weight=sample_weight,
+            )
+        node.set_up(self.train(X_U, y_u, sw_u, depth + 1, title + " - Up"))
+        node.set_down(self.train(X_D, y_d, sw_d, depth + 1, title + " - Down"))
+        return node
+
+    def _build_predictor(self):
+        """Process the leaves to make them predictors"""
+
+        def run_tree(node: Snode):
+            if node.is_leaf():
+                node.make_predictor()
+                return
+            run_tree(node.get_down())
+            run_tree(node.get_up())
+
+        run_tree(self.tree_)
+
+    def _build_clf(self):
+        """Build the correct classifier for the node"""
+        return (
+            LinearSVC(
+                max_iter=self.max_iter,
+                random_state=self.random_state,
+                C=self.C,
+                tol=self.tol,
+            )
+            if self.kernel == "linear"
+            else SVC(
+                kernel=self.kernel,
+                max_iter=self.max_iter,
+                tol=self.tol,
+                C=self.C,
+                gamma=self.gamma,
+                degree=self.degree,
+            )
+        )
+
+    @staticmethod
+    def _reorder_results(y: np.array, indices: np.array) -> np.array:
+        """Reorder an array based on the array of indices passed
+
+        Parameters
+        ----------
+        y : np.array
+            data untidy
+        indices : np.array
+            indices used to set order
+
+        Returns
+        -------
+        np.array
+            array y ordered
+        """
+        # return array of same type given in y
+        y_ordered = y.copy()
+        indices = indices.astype(int)
+        for i, index in enumerate(indices):
+            y_ordered[index] = y[i]
+        return y_ordered
+
+    def predict(self, X: np.array) -> np.array:
+        """Predict labels for each sample in dataset passed
+
+        Parameters
+        ----------
+        X : np.array
+            dataset of samples
+
+        Returns
+        -------
+        np.array
+            array of labels
+
+        Raises
+        ------
+        ValueError
+            if dataset with inconsistent number of features
+        NotFittedError
+            if model is not fitted
+        """
+
+        def predict_class(
+            xp: np.array, indices: np.array, node: Snode
+        ) -> np.array:
+            if xp is None:
+                return [], []
+            if node.is_leaf():
+                # set a class for every sample in dataset
+                prediction = np.full((xp.shape[0], 1), node._class)
+                return prediction, indices
+            self.splitter_.partition(xp, node, train=False)
+            x_u, x_d = self.splitter_.part(xp)
+            i_u, i_d = self.splitter_.part(indices)
+            prx_u, prin_u = predict_class(x_u, i_u, node.get_up())
+            prx_d, prin_d = predict_class(x_d, i_d, node.get_down())
+            return np.append(prx_u, prx_d), np.append(prin_u, prin_d)
+
+        # sklearn check
+        check_is_fitted(self, ["tree_"])
+        # Input validation
+        X = check_array(X)
+        if X.shape[1] != self.n_features_:
+            raise ValueError(
+                f"Expected {self.n_features_} features but got "
+                f"({X.shape[1]})"
+            )
+        # setup prediction & make it happen
+        indices = np.arange(X.shape[0])
+        result = (
+            self._reorder_results(*predict_class(X, indices, self.tree_))
+            .astype(int)
+            .ravel()
+        )
+        return self.classes_[result]
+
+    def score(
+        self, X: np.array, y: np.array, sample_weight: np.array = None
+    ) -> float:
+        """Compute accuracy of the prediction
+
+        Parameters
+        ----------
+        X : np.array
+            dataset of samples to make predictions
+        y : np.array
+            samples labels
+        sample_weight : np.array, optional
+            weights of the samples. Rescale C per sample, by default None
+
+        Returns
+        -------
+        float
+            accuracy of the prediction
+        """
+        # sklearn check
+        check_is_fitted(self)
+        check_classification_targets(y)
+        X, y = check_X_y(X, y)
+        y_pred = self.predict(X).reshape(y.shape)
+        # Compute accuracy for each possible representation
+        _, y_true, y_pred = _check_targets(y, y_pred)
+        check_consistent_length(y_true, y_pred, sample_weight)
+        score = y_true == y_pred
+        return _weighted_sum(score, sample_weight, normalize=True)
+
+    def __iter__(self) -> Siterator:
+        """Create an iterator to be able to visit the nodes of the tree in
+        preorder, can make a list with all the nodes in preorder
+
+        Returns
+        -------
+        Siterator
+            an iterator, can for i in... and list(...)
+        """
+        try:
+            tree = self.tree_
+        except AttributeError:
+            tree = None
+        return Siterator(tree)
+
+    def __str__(self) -> str:
+        """String representation of the tree
+
+        Returns
+        -------
+        str
+            description of nodes in the tree in preorder
+        """
+        output = ""
+        for i in self:
+            output += str(i) + "\n"
+        return output
+
+    def _initialize_max_features(self) -> int:
+        if isinstance(self.max_features, str):
+            if self.max_features == "auto":
+                max_features = max(1, int(np.sqrt(self.n_features_)))
+            elif self.max_features == "sqrt":
+                max_features = max(1, int(np.sqrt(self.n_features_)))
+            elif self.max_features == "log2":
+                max_features = max(1, int(np.log2(self.n_features_)))
+            else:
+                raise ValueError(
+                    "Invalid value for max_features. "
+                    "Allowed string values are 'auto', "
+                    "'sqrt' or 'log2'."
+                )
+        elif self.max_features is None:
+            max_features = self.n_features_
+        elif isinstance(self.max_features, numbers.Integral):
+            max_features = self.max_features
+        else:  # float
+            if self.max_features > 0.0:
+                max_features = max(
+                    1, int(self.max_features * self.n_features_)
+                )
+            else:
+                raise ValueError(
+                    "Invalid value for max_features."
+                    "Allowed float must be in range (0, 1] "
+                    f"got ({self.max_features})"
+                )
+        return max_features

stree/__init__.py

@@ -0,0 +1,3 @@
+from .Strees import Stree, Snode, Siterator, Splitter
+
+__all__ = ["Stree", "Snode", "Siterator", "Splitter"]

stree/tests/Snode_test.py

@@ -0,0 +1,96 @@
+import os
+import unittest
+
+import numpy as np
+
+from stree import Stree, Snode
+from .utils import load_dataset
+
+
+class Snode_test(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        self._random_state = 1
+        self._clf = Stree(random_state=self._random_state)
+        self._clf.fit(*load_dataset(self._random_state))
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def setUp(cls):
+        os.environ["TESTING"] = "1"
+
+    def test_attributes_in_leaves(self):
+        """Check if the attributes in leaves have correct values so they form a
+        predictor
+        """
+
+        def check_leave(node: Snode):
+            if not node.is_leaf():
+                check_leave(node.get_down())
+                check_leave(node.get_up())
+                return
+            # Check Belief in leave
+            classes, card = np.unique(node._y, return_counts=True)
+            max_card = max(card)
+            min_card = min(card)
+            if len(classes) > 1:
+                belief = max_card / (max_card + min_card)
+            else:
+                belief = 1
+            self.assertEqual(belief, node._belief)
+            # Check Class
+            class_computed = classes[card == max_card]
+            self.assertEqual(class_computed, node._class)
+            # Check Partition column
+            self.assertEqual(node._partition_column, -1)
+
+        check_leave(self._clf.tree_)
+
+    def test_nodes_coefs(self):
+        """Check if the nodes of the tree have the right attributes filled"""
+
+        def run_tree(node: Snode):
+            if node._belief < 1:
+                # only exclude pure leaves
+                self.assertIsNotNone(node._clf)
+                self.assertIsNotNone(node._clf.coef_)
+            if node.is_leaf():
+                return
+            run_tree(node.get_up())
+            run_tree(node.get_down())
+
+        model = Stree(self._random_state)
+        model.fit(*load_dataset(self._random_state, 3, 4))
+        run_tree(model.tree_)
+
+    def test_make_predictor_on_leaf(self):
+        test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
+        test.make_predictor()
+        self.assertEqual(1, test._class)
+        self.assertEqual(0.75, test._belief)
+        self.assertEqual(-1, test._partition_column)
+
+    def test_make_predictor_on_not_leaf(self):
+        test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
+        test.set_up(Snode(None, [1], [1], [], 0.0, "another_test"))
+        test.make_predictor()
+        self.assertIsNone(test._class)
+        self.assertEqual(0, test._belief)
+        self.assertEqual(-1, test._partition_column)
+        self.assertEqual(-1, test.get_up()._partition_column)
+
+    def test_make_predictor_on_leaf_bogus_data(self):
+        test = Snode(None, [1, 2, 3, 4], [], [], 0.0, "test")
+        test.make_predictor()
+        self.assertIsNone(test._class)
+        self.assertEqual(-1, test._partition_column)
+
+    def test_copy_node(self):
+        px = [1, 2, 3, 4]
+        py = [1]
+        test = Snode(Stree(), px, py, [], 0.0, "test")
+        computed = Snode.copy(test)
+        self.assertListEqual(computed._X, px)
+        self.assertListEqual(computed._y, py)
+        self.assertEqual("test", computed._title)
+        self.assertIsInstance(computed._clf, Stree)
+        self.assertEqual(test._partition_column, computed._partition_column)

stree/tests/Splitter_test.py

@@ -0,0 +1,224 @@
+import os
+import unittest
+import random
+
+import numpy as np
+from sklearn.svm import SVC
+from sklearn.datasets import load_wine, load_iris
+from stree import Splitter
+
+
+class Splitter_test(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        self._random_state = 1
+        super().__init__(*args, **kwargs)
+
+    @staticmethod
+    def build(
+        clf=SVC,
+        min_samples_split=0,
+        splitter_type="random",
+        criterion="gini",
+        criteria="max_samples",
+        random_state=None,
+    ):
+        return Splitter(
+            clf=clf(random_state=random_state, kernel="rbf"),
+            min_samples_split=min_samples_split,
+            splitter_type=splitter_type,
+            criterion=criterion,
+            criteria=criteria,
+            random_state=random_state,
+        )
+
+    @classmethod
+    def setUp(cls):
+        os.environ["TESTING"] = "1"
+
+    def test_init(self):
+        with self.assertRaises(ValueError):
+            self.build(criterion="duck")
+        with self.assertRaises(ValueError):
+            self.build(splitter_type="duck")
+        with self.assertRaises(ValueError):
+            self.build(criteria="duck")
+        with self.assertRaises(ValueError):
+            _ = Splitter(clf=None)
+        for splitter_type in ["best", "random"]:
+            for criterion in ["gini", "entropy"]:
+                for criteria in ["max_samples", "impurity"]:
+                    tcl = self.build(
+                        splitter_type=splitter_type,
+                        criterion=criterion,
+                        criteria=criteria,
+                    )
+                    self.assertEqual(splitter_type, tcl._splitter_type)
+                    self.assertEqual(criterion, tcl._criterion)
+                    self.assertEqual(criteria, tcl._criteria)
+
+    def test_gini(self):
+        expected_values = [
+            ([0, 1, 1, 1, 1, 1, 0, 0, 0, 1], 0.48),
+            ([0, 1, 1, 2, 2, 3, 4, 5, 3, 2, 1, 1], 0.7777777777777778),
+            ([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2], 0.520408163265306),
+            ([0, 0, 1, 1, 1, 1, 0, 0], 0.5),
+            ([0, 0, 1, 1, 2, 2, 3, 3], 0.75),
+            ([0, 0, 1, 1, 1, 1, 1, 1], 0.375),
+            ([0], 0),
+            ([1, 1, 1, 1], 0),
+        ]
+        for labels, expected in expected_values:
+            self.assertAlmostEqual(expected, Splitter._gini(labels))
+            tcl = self.build(criterion="gini")
+            self.assertAlmostEqual(expected, tcl.criterion_function(labels))
+
+    def test_entropy(self):
+        expected_values = [
+            ([0, 1, 1, 1, 1, 1, 0, 0, 0, 1], 0.9709505944546686),
+            ([0, 1, 1, 2, 2, 3, 4, 5, 3, 2, 1, 1], 0.9111886696810589),
+            ([0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 2], 0.8120406807940999),
+            ([0, 0, 1, 1, 1, 1, 0, 0], 1),
+            ([0, 0, 1, 1, 2, 2, 3, 3], 1),
+            ([0, 0, 1, 1, 1, 1, 1, 1], 0.8112781244591328),
+            ([1], 0),
+            ([0, 0, 0, 0], 0),
+        ]
+        for labels, expected in expected_values:
+            self.assertAlmostEqual(expected, Splitter._entropy(labels))
+            tcl = self.build(criterion="entropy")
+            self.assertAlmostEqual(expected, tcl.criterion_function(labels))
+
+    def test_information_gain(self):
+        expected_values = [
+            (
+                [0, 1, 1, 1, 1, 1],
+                [0, 0, 0, 1],
+                0.16333333333333333,
+                0.25642589168200297,
+            ),
+            (
+                [0, 1, 1, 2, 2, 3, 4, 5, 3, 2, 1, 1],
+                [5, 3, 2, 1, 1],
+                0.007381776239907684,
+                -0.03328610916207225,
+            ),
+            ([], [], 0.0, 0.0),
+            ([1], [], 0.0, 0.0),
+            ([], [1], 0.0, 0.0),
+            ([0, 0, 0, 0], [0, 0], 0.0, 0.0),
+            ([], [1, 1, 1, 2], 0.0, 0.0),
+            (None, [1, 2, 3], 0.0, 0.0),
+            ([1, 2, 3], None, 0.0, 0.0),
+        ]
+        for yu, yd, expected_gini, expected_entropy in expected_values:
+            yu = np.array(yu, dtype=np.int32) if yu is not None else None
+            yd = np.array(yd, dtype=np.int32) if yd is not None else None
+            if yu is not None and yd is not None:
+                complete = np.append(yu, yd)
+            elif yd is not None:
+                complete = yd
+            else:
+                complete = yu
+            tcl = self.build(criterion="gini")
+            computed = tcl.information_gain(complete, yu, yd)
+            self.assertAlmostEqual(expected_gini, computed)
+            tcl = self.build(criterion="entropy")
+            computed = tcl.information_gain(complete, yu, yd)
+            self.assertAlmostEqual(expected_entropy, computed)
+
+    def test_max_samples(self):
+        tcl = self.build(criteria="max_samples")
+        data = np.array(
+            [
+                [-0.1, 0.2, -0.3],
+                [0.7, 0.01, -0.1],
+                [0.7, -0.9, 0.5],
+                [0.1, 0.2, 0.3],
+                [-0.1, 0.2, 0.3],
+                [-0.1, 0.2, 0.3],
+            ]
+        )
+        expected = data[:, 0]
+        y = [1, 2, 1, 0, 0, 0]
+        computed = tcl._max_samples(data, y)
+        self.assertEqual(0, computed)
+        computed_data = data[:, computed]
+        self.assertEqual((6,), computed_data.shape)
+        self.assertListEqual(expected.tolist(), computed_data.tolist())
+
+    def test_impurity(self):
+        tcl = self.build(criteria="impurity")
+        data = np.array(
+            [
+                [-0.1, 0.2, -0.3],
+                [0.7, 0.01, -0.1],
+                [0.7, -0.9, 0.5],
+                [0.1, 0.2, 0.3],
+                [-0.1, 0.2, 0.3],
+                [-0.1, 0.2, 0.3],
+            ]
+        )
+        expected = data[:, 2]
+        y = np.array([1, 2, 1, 0, 0, 0])
+        computed = tcl._impurity(data, y)
+        self.assertEqual(2, computed)
+        computed_data = data[:, computed]
+        self.assertEqual((6,), computed_data.shape)
+        self.assertListEqual(expected.tolist(), computed_data.tolist())
+
+    def test_generate_subspaces(self):
+        features = 250
+        for max_features in range(2, features):
+            num = len(Splitter._generate_spaces(features, max_features))
+            self.assertEqual(5, num)
+        self.assertEqual(3, len(Splitter._generate_spaces(3, 2)))
+        self.assertEqual(4, len(Splitter._generate_spaces(4, 3)))
+
+    def test_best_splitter_few_sets(self):
+        X, y = load_iris(return_X_y=True)
+        X = np.delete(X, 3, 1)
+        tcl = self.build(splitter_type="best", random_state=self._random_state)
+        dataset, computed = tcl.get_subspace(X, y, max_features=2)
+        self.assertListEqual([0, 2], list(computed))
+        self.assertListEqual(X[:, computed].tolist(), dataset.tolist())
+
+    def test_splitter_parameter(self):
+        expected_values = [
+            [1, 4, 9, 12],  # best   entropy max_samples
+            [1, 3, 6, 10],  # best   entropy impurity
+            [6, 8, 10, 12],  # best   gini    max_samples
+            [7, 8, 10, 11],  # best   gini    impurity
+            [0, 3, 8, 12],  # random entropy max_samples
+            [0, 3, 9, 11],  # random entropy impurity
+            [0, 4, 7, 12],  # random gini    max_samples
+            [0, 2, 5, 6],  # random gini    impurity
+        ]
+        X, y = load_wine(return_X_y=True)
+        rn = 0
+        for splitter_type in ["best", "random"]:
+            for criterion in ["entropy", "gini"]:
+                for criteria in [
+                    "max_samples",
+                    "impurity",
+                ]:
+                    tcl = self.build(
+                        splitter_type=splitter_type,
+                        criterion=criterion,
+                        criteria=criteria,
+                    )
+                    expected = expected_values.pop(0)
+                    random.seed(rn)
+                    rn += 1
+                    dataset, computed = tcl.get_subspace(X, y, max_features=4)
+                    # print(
+                    #     "{},  # {:7s}{:8s}{:15s}".format(
+                    #         list(computed),
+                    #         splitter_type,
+                    #         criterion,
+                    #         criteria,
+                    #     )
+                    # )
+                    self.assertListEqual(expected, list(computed))
+                    self.assertListEqual(
+                        X[:, computed].tolist(), dataset.tolist()
+                    )

stree/tests/Stree_test.py

@@ -0,0 +1,441 @@
+import os
+import unittest
+import warnings
+
+import numpy as np
+from sklearn.datasets import load_iris, load_wine
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.svm import LinearSVC
+
+from stree import Stree, Snode
+from .utils import load_dataset
+
+
+class Stree_test(unittest.TestCase):
+    def __init__(self, *args, **kwargs):
+        self._random_state = 1
+        self._kernels = ["linear", "rbf", "poly"]
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def setUp(cls):
+        os.environ["TESTING"] = "1"
+
+    def _check_tree(self, node: Snode):
+        """Check recursively that the nodes that are not leaves have the
+        correct number of labels and its sons have the right number of elements
+        in their dataset
+
+        Parameters
+        ----------
+        node : Snode
+            node to check
+        """
+        if node.is_leaf():
+            return
+        y_prediction = node._clf.predict(node._X)
+        y_down = node.get_down()._y
+        y_up = node.get_up()._y
+        # Is a correct partition in terms of cadinality?
+        # i.e. The partition algorithm didn't forget any sample
+        self.assertEqual(node._y.shape[0], y_down.shape[0] + y_up.shape[0])
+        unique_y, count_y = np.unique(node._y, return_counts=True)
+        _, count_d = np.unique(y_down, return_counts=True)
+        _, count_u = np.unique(y_up, return_counts=True)
+        #
+        for i in unique_y:
+            number_up = count_u[i]
+            try:
+                number_down = count_d[i]
+            except IndexError:
+                number_down = 0
+            self.assertEqual(count_y[i], number_down + number_up)
+        # Is the partition made the same as the prediction?
+        # as the node is not a leaf...
+        _, count_yp = np.unique(y_prediction, return_counts=True)
+        self.assertEqual(count_yp[1], y_up.shape[0])
+        self.assertEqual(count_yp[0], y_down.shape[0])
+        self._check_tree(node.get_down())
+        self._check_tree(node.get_up())
+
+    def test_build_tree(self):
+        """Check if the tree is built the same way as predictions of models"""
+        warnings.filterwarnings("ignore")
+        for kernel in self._kernels:
+            clf = Stree(kernel=kernel, random_state=self._random_state)
+            clf.fit(*load_dataset(self._random_state))
+            self._check_tree(clf.tree_)
+
+    def test_single_prediction(self):
+        X, y = load_dataset(self._random_state)
+        for kernel in self._kernels:
+            clf = Stree(kernel=kernel, random_state=self._random_state)
+            yp = clf.fit(X, y).predict((X[0, :].reshape(-1, X.shape[1])))
+            self.assertEqual(yp[0], y[0])
+
+    def test_multiple_prediction(self):
+        # First 27 elements the predictions are the same as the truth
+        num = 27
+        X, y = load_dataset(self._random_state)
+        for kernel in self._kernels:
+            clf = Stree(kernel=kernel, random_state=self._random_state)
+            yp = clf.fit(X, y).predict(X[:num, :])
+            self.assertListEqual(y[:num].tolist(), yp.tolist())
+
+    def test_single_vs_multiple_prediction(self):
+        """Check if predicting sample by sample gives the same result as
+        predicting all samples at once
+        """
+        X, y = load_dataset(self._random_state)
+        for kernel in self._kernels:
+            clf = Stree(kernel=kernel, random_state=self._random_state)
+            clf.fit(X, y)
+            # Compute prediction line by line
+            yp_line = np.array([], dtype=int)
+            for xp in X:
+                yp_line = np.append(
+                    yp_line, clf.predict(xp.reshape(-1, X.shape[1]))
+                )
+            # Compute prediction at once
+            yp_once = clf.predict(X)
+            self.assertListEqual(yp_line.tolist(), yp_once.tolist())
+
+    def test_iterator_and_str(self):
+        """Check preorder iterator"""
+        expected = [
+            "root feaures=(0, 1, 2) impurity=1.0000 counts=(array([0, 1]), arr"
+            "ay([750, 750]))",
+            "root - Down, <cgaf> - Leaf class=0 belief= 0.928297 impurity=0.37"
+            "22 counts=(array([0, 1]), array([725,  56]))",
+            "root - Up feaures=(0, 1, 2) impurity=0.2178 counts=(array([0, 1])"
+            ", array([ 25, 694]))",
+            "root - Up - Down feaures=(0, 1, 2) impurity=0.8454 counts=(array("
+            "[0, 1]), array([8, 3]))",
+            "root - Up - Down - Down, <pure> - Leaf class=0 belief= 1.000000 i"
+            "mpurity=0.0000 counts=(array([0]), array([7]))",
+            "root - Up - Down - Up, <cgaf> - Leaf class=1 belief= 0.750000 imp"
+            "urity=0.8113 counts=(array([0, 1]), array([1, 3]))",
+            "root - Up - Up, <cgaf> - Leaf class=1 belief= 0.975989 impurity=0"
+            ".1634 counts=(array([0, 1]), array([ 17, 691]))",
+        ]
+        computed = []
+        expected_string = ""
+        clf = Stree(kernel="linear", random_state=self._random_state)
+        clf.fit(*load_dataset(self._random_state))
+        for node in clf:
+            computed.append(str(node))
+            expected_string += str(node) + "\n"
+        self.assertListEqual(expected, computed)
+        self.assertEqual(expected_string, str(clf))
+
+    @staticmethod
+    def test_is_a_sklearn_classifier():
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        warnings.filterwarnings("ignore", category=RuntimeWarning)
+        from sklearn.utils.estimator_checks import check_estimator
+
+        check_estimator(Stree())
+
+    def test_exception_if_C_is_negative(self):
+        tclf = Stree(C=-1)
+        with self.assertRaises(ValueError):
+            tclf.fit(*load_dataset(self._random_state))
+
+    def test_exception_if_bogus_split_criteria(self):
+        tclf = Stree(split_criteria="duck")
+        with self.assertRaises(ValueError):
+            tclf.fit(*load_dataset(self._random_state))
+
+    def test_check_max_depth_is_positive_or_None(self):
+        tcl = Stree()
+        self.assertIsNone(tcl.max_depth)
+        tcl = Stree(max_depth=1)
+        self.assertGreaterEqual(1, tcl.max_depth)
+        with self.assertRaises(ValueError):
+            tcl = Stree(max_depth=-1)
+            tcl.fit(*load_dataset(self._random_state))
+
+    def test_check_max_depth(self):
+        depths = (3, 4)
+        for depth in depths:
+            tcl = Stree(random_state=self._random_state, max_depth=depth)
+            tcl.fit(*load_dataset(self._random_state))
+            self.assertEqual(depth, tcl.depth_)
+
+    def test_unfitted_tree_is_iterable(self):
+        tcl = Stree()
+        self.assertEqual(0, len(list(tcl)))
+
+    def test_min_samples_split(self):
+        dataset = [[1], [2], [3]], [1, 1, 0]
+        tcl_split = Stree(min_samples_split=3).fit(*dataset)
+        self.assertIsNotNone(tcl_split.tree_.get_down())
+        self.assertIsNotNone(tcl_split.tree_.get_up())
+        tcl_nosplit = Stree(min_samples_split=4).fit(*dataset)
+        self.assertIsNone(tcl_nosplit.tree_.get_down())
+        self.assertIsNone(tcl_nosplit.tree_.get_up())
+
+    def test_simple_muticlass_dataset(self):
+        for kernel in self._kernels:
+            clf = Stree(
+                kernel=kernel,
+                split_criteria="max_samples",
+                random_state=self._random_state,
+            )
+            px = [[1, 2], [5, 6], [9, 10]]
+            py = [0, 1, 2]
+            clf.fit(px, py)
+            self.assertEqual(1.0, clf.score(px, py))
+            self.assertListEqual(py, clf.predict(px).tolist())
+            self.assertListEqual(py, clf.classes_.tolist())
+
+    def test_muticlass_dataset(self):
+        datasets = {
+            "Synt": load_dataset(random_state=self._random_state, n_classes=3),
+            "Iris": load_wine(return_X_y=True),
+        }
+        outcomes = {
+            "Synt": {
+                "max_samples linear": 0.9606666666666667,
+                "max_samples rbf": 0.7133333333333334,
+                "max_samples poly": 0.49066666666666664,
+                "impurity linear": 0.9606666666666667,
+                "impurity rbf": 0.7133333333333334,
+                "impurity poly": 0.49066666666666664,
+            },
+            "Iris": {
+                "max_samples linear": 1.0,
+                "max_samples rbf": 0.6910112359550562,
+                "max_samples poly": 0.6966292134831461,
+                "impurity linear": 1,
+                "impurity rbf": 0.6910112359550562,
+                "impurity poly": 0.6966292134831461,
+            },
+        }
+
+        for name, dataset in datasets.items():
+            px, py = dataset
+            for criteria in ["max_samples", "impurity"]:
+                for kernel in self._kernels:
+                    clf = Stree(
+                        C=55,
+                        max_iter=1e5,
+                        kernel=kernel,
+                        random_state=self._random_state,
+                    )
+                    clf.fit(px, py)
+                    outcome = outcomes[name][f"{criteria} {kernel}"]
+                    # print(
+                    #     f"{name} {criteria} {kernel} {outcome} {clf.score(px"
+                    #     ", py)}"
+                    # )
+                    self.assertAlmostEqual(outcome, clf.score(px, py))
+
+    def test_max_features(self):
+        n_features = 16
+        expected_values = [
+            ("auto", 4),
+            ("log2", 4),
+            ("sqrt", 4),
+            (0.5, 8),
+            (3, 3),
+            (None, 16),
+        ]
+        clf = Stree()
+        clf.n_features_ = n_features
+        for max_features, expected in expected_values:
+            clf.set_params(**dict(max_features=max_features))
+            computed = clf._initialize_max_features()
+            self.assertEqual(expected, computed)
+        # Check bogus max_features
+        values = ["duck", -0.1, 0.0]
+        for max_features in values:
+            clf.set_params(**dict(max_features=max_features))
+            with self.assertRaises(ValueError):
+                _ = clf._initialize_max_features()
+
+    def test_get_subspaces(self):
+        dataset = np.random.random((10, 16))
+        y = np.random.randint(0, 2, 10)
+        expected_values = [
+            ("auto", 4),
+            ("log2", 4),
+            ("sqrt", 4),
+            (0.5, 8),
+            (3, 3),
+            (None, 16),
+        ]
+        clf = Stree()
+        for max_features, expected in expected_values:
+            clf.set_params(**dict(max_features=max_features))
+            clf.fit(dataset, y)
+            computed, indices = clf.splitter_.get_subspace(
+                dataset, y, clf.max_features_
+            )
+            self.assertListEqual(
+                dataset[:, indices].tolist(), computed.tolist()
+            )
+            self.assertEqual(expected, len(indices))
+
+    def test_bogus_criterion(self):
+        clf = Stree(criterion="duck")
+        with self.assertRaises(ValueError):
+            clf.fit(*load_dataset())
+
+    def test_predict_feature_dimensions(self):
+        X = np.random.rand(10, 5)
+        y = np.random.randint(0, 2, 10)
+        clf = Stree()
+        clf.fit(X, y)
+        with self.assertRaises(ValueError):
+            clf.predict(X[:, :3])
+
+    # Tests of score
+
+    def test_score_binary(self):
+        X, y = load_dataset(self._random_state)
+        accuracies = [
+            0.9506666666666667,
+            0.9606666666666667,
+            0.9433333333333334,
+        ]
+        for kernel, accuracy_expected in zip(self._kernels, accuracies):
+            clf = Stree(
+                random_state=self._random_state,
+                kernel=kernel,
+            )
+            clf.fit(X, y)
+            accuracy_score = clf.score(X, y)
+            yp = clf.predict(X)
+            accuracy_computed = np.mean(yp == y)
+            self.assertEqual(accuracy_score, accuracy_computed)
+            self.assertAlmostEqual(accuracy_expected, accuracy_score)
+
+    def test_score_max_features(self):
+        X, y = load_dataset(self._random_state)
+        clf = Stree(random_state=self._random_state, max_features=2)
+        clf.fit(X, y)
+        self.assertAlmostEqual(0.9246666666666666, clf.score(X, y))
+
+    def test_bogus_splitter_parameter(self):
+        clf = Stree(splitter="duck")
+        with self.assertRaises(ValueError):
+            clf.fit(*load_dataset())
+
+    def test_multiclass_classifier_integrity(self):
+        """Checks if the multiclass operation is done right"""
+        X, y = load_iris(return_X_y=True)
+        clf = Stree(random_state=0)
+        clf.fit(X, y)
+        score = clf.score(X, y)
+        # Check accuracy of the whole model
+        self.assertAlmostEquals(0.98, score, 5)
+        svm = LinearSVC(random_state=0)
+        svm.fit(X, y)
+        self.assertAlmostEquals(0.9666666666666667, svm.score(X, y), 5)
+        data = svm.decision_function(X)
+        expected = [
+            0.4444444444444444,
+            0.35777777777777775,
+            0.4569777777777778,
+        ]
+        ty = data.copy()
+        ty[data <= 0] = 0
+        ty[data > 0] = 1
+        ty = ty.astype(int)
+        for i in range(3):
+            self.assertAlmostEquals(
+                expected[i],
+                clf.splitter_._gini(ty[:, i]),
+            )
+        # 1st Branch
+        # up has to have 50 samples of class 0
+        # down should have 100 [50, 50]
+        up = data[:, 2] > 0
+        resup = np.unique(y[up], return_counts=True)
+        resdn = np.unique(y[~up], return_counts=True)
+        self.assertListEqual([1, 2], resup[0].tolist())
+        self.assertListEqual([3, 50], resup[1].tolist())
+        self.assertListEqual([0, 1], resdn[0].tolist())
+        self.assertListEqual([50, 47], resdn[1].tolist())
+        # 2nd Branch
+        # up  should have 53 samples of classes [1, 2] [3, 50]
+        # down shoud have 47 samples of class 1
+        node_up = clf.tree_.get_down().get_up()
+        node_dn = clf.tree_.get_down().get_down()
+        resup = np.unique(node_up._y, return_counts=True)
+        resdn = np.unique(node_dn._y, return_counts=True)
+        self.assertListEqual([1, 2], resup[0].tolist())
+        self.assertListEqual([3, 50], resup[1].tolist())
+        self.assertListEqual([1], resdn[0].tolist())
+        self.assertListEqual([47], resdn[1].tolist())
+
+    def test_score_multiclass_rbf(self):
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=500,
+        )
+        clf = Stree(kernel="rbf", random_state=self._random_state)
+        self.assertEqual(0.824, clf.fit(X, y).score(X, y))
+        X, y = load_wine(return_X_y=True)
+        self.assertEqual(0.6741573033707865, clf.fit(X, y).score(X, y))
+
+    def test_score_multiclass_poly(self):
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=500,
+        )
+        clf = Stree(
+            kernel="poly", random_state=self._random_state, C=10, degree=5
+        )
+        self.assertEqual(0.786, clf.fit(X, y).score(X, y))
+        X, y = load_wine(return_X_y=True)
+        self.assertEqual(0.702247191011236, clf.fit(X, y).score(X, y))
+
+    def test_score_multiclass_linear(self):
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=1500,
+        )
+        clf = Stree(kernel="linear", random_state=self._random_state)
+        self.assertEqual(0.9533333333333334, clf.fit(X, y).score(X, y))
+        X, y = load_wine(return_X_y=True)
+        self.assertEqual(0.9550561797752809, clf.fit(X, y).score(X, y))
+
+    def test_zero_all_sample_weights(self):
+        X, y = load_dataset(self._random_state)
+        with self.assertRaises(ValueError):
+            Stree().fit(X, y, np.zeros(len(y)))
+
+    def test_mask_samples_weighted_zero(self):
+        X = np.array(
+            [
+                [1, 1],
+                [1, 1],
+                [1, 1],
+                [2, 2],
+                [2, 2],
+                [2, 2],
+                [3, 3],
+                [3, 3],
+                [3, 3],
+            ]
+        )
+        y = np.array([1, 1, 1, 2, 2, 2, 5, 5, 5])
+        yw = np.array([1, 1, 1, 5, 5, 5, 5, 5, 5])
+        w = [1, 1, 1, 0, 0, 0, 1, 1, 1]
+        model1 = Stree().fit(X, y)
+        model2 = Stree().fit(X, y, w)
+        predict1 = model1.predict(X)
+        predict2 = model2.predict(X)
+        self.assertListEqual(y.tolist(), predict1.tolist())
+        self.assertListEqual(yw.tolist(), predict2.tolist())
+        self.assertEqual(model1.score(X, y), 1)
+        self.assertAlmostEqual(model2.score(X, y), 0.66666667)
+        self.assertEqual(model2.score(X, y, w), 1)

stree/tests/__init__.py

@@ -0,0 +1,5 @@
+from .Stree_test import Stree_test
+from .Snode_test import Snode_test
+from .Splitter_test import Splitter_test
+
+__all__ = ["Stree_test", "Snode_test", "Splitter_test"]

stree/tests/utils.py

@@ -0,0 +1,17 @@
+from sklearn.datasets import make_classification
+
+
+def load_dataset(random_state=0, n_classes=2, n_features=3, n_samples=1500):
+    X, y = make_classification(
+        n_samples=n_samples,
+        n_features=n_features,
+        n_informative=3,
+        n_redundant=0,
+        n_repeated=0,
+        n_classes=n_classes,
+        n_clusters_per_class=2,
+        class_sep=1.5,
+        flip_y=0,
+        random_state=random_state,
+    )
+    return X, y

test2.ipynb

@@ -1,249 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "import pandas as pd\n",
-    "from sklearn.svm import LinearSVC\n",
-    "from sklearn.tree import DecisionTreeClassifier\n",
-    "from sklearn.datasets import make_classification, load_iris, load_wine\n",
-    "from trees.Stree import Stree\n",
-    "import time"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "if not os.path.isfile('data/creditcard.csv'):\n",
-    "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
-    "    !tar xzf creditcard.tgz"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stdout",
-     "text": "Fraud: 0.173% 492\nValid: 99.827% 284315\nX.shape (1492, 28)  y.shape (1492,)\nFraud: 32.976% 492\nValid: 67.024% 1000\n"
-    }
-   ],
-   "source": [
-    "import time\n",
-    "from sklearn.model_selection import train_test_split\n",
-    "from trees.Stree import Stree\n",
-    "\n",
-    "random_state=1\n",
-    "\n",
-    "def load_creditcard(n_examples=0):\n",
-    "    import pandas as pd\n",
-    "    import numpy as np\n",
-    "    import random\n",
-    "    df = pd.read_csv('data/creditcard.csv')\n",
-    "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
-    "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
-    "    y = df.Class\n",
-    "    X = df.drop(['Class', 'Time', 'Amount'], axis=1).values\n",
-    "    if n_examples > 0:\n",
-    "        # Take first n_examples samples\n",
-    "        X = X[:n_examples, :]\n",
-    "        y = y[:n_examples, :]\n",
-    "    else:\n",
-    "        # Take all the positive samples with a number of random negatives\n",
-    "        if n_examples < 0:\n",
-    "            Xt = X[(y == 1).ravel()]\n",
-    "            yt = y[(y == 1).ravel()]\n",
-    "            indices = random.sample(range(X.shape[0]), -1 * n_examples)\n",
-    "            X = np.append(Xt, X[indices], axis=0)\n",
-    "            y = np.append(yt, y[indices], axis=0)\n",
-    "    print(\"X.shape\", X.shape, \" y.shape\", y.shape)\n",
-    "    print(\"Fraud: {0:.3f}% {1}\".format(len(y[y == 1])*100/X.shape[0], len(y[y == 1])))\n",
-    "    print(\"Valid: {0:.3f}% {1}\".format(len(y[y == 0]) * 100 / X.shape[0], len(y[y == 0])))\n",
-    "    Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7, shuffle=True, random_state=random_state, stratify=y)\n",
-    "    return Xtrain, Xtest, ytrain, ytest\n",
-    "\n",
-    "# data = load_creditcard(-5000) # Take all true samples + 5000 of the others\n",
-    "# data = load_creditcard(5000)  # Take the first 5000 samples\n",
-    "data = load_creditcard(-1000) # Take all the samples\n",
-    "\n",
-    "Xtrain = data[0]\n",
-    "Xtest = data[1]\n",
-    "ytrain = data[2]\n",
-    "ytest = data[3]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stdout",
-     "text": "************** C=0.001 ****************************\nClassifier's accuracy (train): 0.9550\nClassifier's accuracy (test) : 0.9487\nroot\nroot - Down\nroot - Down - Down, <cgaf> - Leaf class=1 belief=0.977346 counts=(array([0, 1]), array([  7, 302]))\nroot - Up\nroot - Up - Down, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\nroot - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\nroot - Up - Up\nroot - Up - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([2]))\nroot - Up - Up - Up, <cgaf> - Leaf class=0 belief=0.945280 counts=(array([0, 1]), array([691,  40]))\n\n**************************************************\n************** C=0.01 ****************************\nClassifier's accuracy (train): 0.9569\nClassifier's accuracy (test) : 0.9576\nroot\nroot - Down, <cgaf> - Leaf class=1 belief=0.986971 counts=(array([0, 1]), array([  4, 303]))\nroot - Up, <cgaf> - Leaf class=0 belief=0.944369 counts=(array([0, 1]), array([696,  41]))\n\n**************************************************\n************** C=1 ****************************\nClassifier's accuracy (train): 0.9674\nClassifier's accuracy (test) : 0.9554\nroot\nroot - Down\nroot - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([310]))\nroot - Up, <cgaf> - Leaf class=0 belief=0.953232 counts=(array([0, 1]), array([693,  34]))\nroot - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([7]))\n\n**************************************************\n************** C=5 ****************************\nClassifier's accuracy (train): 0.9693\nClassifier's accuracy (test) : 0.9487\nroot\nroot - Down\nroot - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([310]))\nroot - Up\nroot - Up - Down, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\nroot - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([7]))\nroot - Up - Up\nroot - Up - Up - Down, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([2]))\nroot - Up - Up - Up\nroot - Up - Up - Up - Down, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([2]))\nroot - Up - Up - Up - Up\nroot - Up - Up - Up - Up - Down\nroot - Up - Up - Up - Up - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([2]))\nroot - Up - Up - Up - Up - Up, <cgaf> - Leaf class=0 belief=0.955494 counts=(array([0, 1]), array([687,  32]))\nroot - Up - Up - Up - Up - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\n\n**************************************************\n************** C=17 ****************************\nClassifier's accuracy (train): 0.9780\nClassifier's accuracy (test) : 0.9487\nroot\nroot - Down\nroot - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([301]))\nroot - Up\nroot - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([2]))\nroot - Down - Up\nroot - Down - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([15]))\nroot - Up - Up\nroot - Up - Up - Down\nroot - Up - Up - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([3]))\nroot - Down - Up - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([15]))\nroot - Up - Up - Up, <cgaf> - Leaf class=0 belief=0.967468 counts=(array([0, 1]), array([684,  23]))\nroot - Up - Up - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\n\n**************************************************\n0.7277 secs\n"
-    }
-   ],
-   "source": [
-    "t = time.time()\n",
-    "for C in (.001, .01, 1, 5, 17):\n",
-    "    clf = Stree(C=C, random_state=random_state)\n",
-    "    clf.fit(Xtrain, ytrain)\n",
-    "    print(f\"************** C={C} ****************************\")\n",
-    "    print(f\"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}\")\n",
-    "    print(f\"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}\")\n",
-    "    print(clf)\n",
-    "    print(f\"**************************************************\")\n",
-    "print(f\"{time.time() - t:.4f} secs\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np\n",
-    "from sklearn.preprocessing import StandardScaler\n",
-    "from sklearn.svm import LinearSVC\n",
-    "from sklearn.calibration import CalibratedClassifierCV\n",
-    "scaler = StandardScaler()\n",
-    "cclf = CalibratedClassifierCV(base_estimator=LinearSVC(), cv=5)\n",
-    "cclf.fit(Xtrain, ytrain)\n",
-    "res = cclf.predict_proba(Xtest)\n",
-    "#an array containing probabilities of belonging to the 1st class"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stdout",
-     "text": "root\nroot - Down\nroot - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([301]))\nroot - Up\nroot - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([2]))\nroot - Down - Up\nroot - Down - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([15]))\nroot - Up - Up\nroot - Up - Up - Down\nroot - Up - Up - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([3]))\nroot - Down - Up - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([15]))\nroot - Up - Up - Up, <cgaf> - Leaf class=0 belief=0.967468 counts=(array([0, 1]), array([684,  23]))\nroot - Up - Up - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\n"
-    }
-   ],
-   "source": [
-    "for i in list(clf):\n",
-    "    print(i)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stdout",
-     "text": "root\nroot - Down\nroot - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([301]))\nroot - Up\nroot - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([2]))\nroot - Down - Up\nroot - Down - Up - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([15]))\nroot - Up - Up\nroot - Up - Up - Down\nroot - Up - Up - Down - Down, <pure> - Leaf class=1 belief=1.000000 counts=(array([1]), array([3]))\nroot - Down - Up - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([15]))\nroot - Up - Up - Up, <cgaf> - Leaf class=0 belief=0.967468 counts=(array([0, 1]), array([684,  23]))\nroot - Up - Up - Down - Up, <pure> - Leaf class=0 belief=1.000000 counts=(array([0]), array([1]))\n"
-    }
-   ],
-   "source": [
-    "for i in clf:\n",
-    "    print(i)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "metadata": {},
-   "outputs": [
-    {
-     "output_type": "display_data",
-     "data": {
-      "text/plain": "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …",
-      "application/vnd.jupyter.widget-view+json": {
-       "version_major": 2,
-       "version_minor": 0,
-       "model_id": "0025f832c1734afc944021e5990c2d11"
-      }
-     },
-     "metadata": {}
-    }
-   ],
-   "source": [
-    "%matplotlib widget\n",
-    "from mpl_toolkits.mplot3d import Axes3D\n",
-    "import matplotlib.pyplot as plt\n",
-    "from matplotlib import cm\n",
-    "from matplotlib.ticker import LinearLocator, FormatStrFormatter\n",
-    "import numpy as np\n",
-    "\n",
-    "fig = plt.figure()\n",
-    "ax = fig.gca(projection='3d')\n",
-    "\n",
-    "scale = 8\n",
-    "# Make data.\n",
-    "X = np.arange(-scale, scale, 0.25)\n",
-    "Y = np.arange(-scale, scale, 0.25)\n",
-    "X, Y = np.meshgrid(X, Y)\n",
-    "Z = X**2 + Y**2\n",
-    "\n",
-    "# Plot the surface.\n",
-    "surf = ax.plot_surface(X, Y, Z, cmap=cm.coolwarm,\n",
-    "                   linewidth=0, antialiased=False)\n",
-    "\n",
-    "# Customize the z axis.\n",
-    "ax.set_zlim(0, 100)\n",
-    "ax.zaxis.set_major_locator(LinearLocator(10))\n",
-    "ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))\n",
-    "\n",
-    "# rotate the axes and update\n",
-    "#for angle in range(0, 360):\n",
-    "#   ax.view_init(30, 40)\n",
-    "\n",
-    "# Add a color bar which maps values to colors.\n",
-    "fig.colorbar(surf, shrink=0.5, aspect=5)\n",
-    "\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.7.6-final"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

tests/Snode_test.py

@@ -1,72 +0,0 @@
-import os
-import unittest
-
-import numpy as np
-from sklearn.datasets import make_classification
-
-from trees.Stree import Stree, Snode
-
-
-class Snode_test(unittest.TestCase):
-
-    def __init__(self, *args, **kwargs):
-        os.environ['TESTING'] = '1'
-        self._random_state = 1
-        self._clf = Stree(random_state=self._random_state,
-                            use_predictions=True)
-        self._clf.fit(*self._get_Xy())
-        super(Snode_test, self).__init__(*args, **kwargs)
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            os.environ.pop('TESTING')
-        except:
-            pass
-
-    def _get_Xy(self):
-        X, y = make_classification(n_samples=1500, n_features=3, n_informative=3,
-                                   n_redundant=0, n_repeated=0, n_classes=2, n_clusters_per_class=2,
-                                   class_sep=1.5, flip_y=0, weights=[0.5, 0.5], random_state=self._random_state)
-        return X, y
-
-    def test_attributes_in_leaves(self):
-        """Check if the attributes in leaves have correct values so they form a predictor
-        """
-        def check_leave(node: Snode):
-            if not node.is_leaf():
-                check_leave(node.get_down())
-                check_leave(node.get_up())
-                return
-            # Check Belief in leave
-            classes, card = np.unique(node._y, return_counts=True)
-            max_card = max(card)
-            min_card = min(card)
-            if len(classes) > 1:
-                try:
-                    belief = max_card / (max_card + min_card)
-                except:
-                    belief = 0.
-            else:
-                belief = 1
-            self.assertEqual(belief, node._belief)
-            # Check Class
-            class_computed = classes[card == max_card]
-            self.assertEqual(class_computed, node._class)
-        check_leave(self._clf._tree)
-    
-    def test_nodes_coefs(self):
-        """Check if the nodes of the tree have the right attributes filled
-        """
-        def run_tree(node: Snode):
-            if node._belief < 1:
-                # only exclude pure leaves
-                self.assertIsNotNone(node._clf)
-                self.assertIsNotNone(node._clf.coef_)
-                self.assertIsNotNone(node._vector)
-                self.assertIsNotNone(node._interceptor)
-            if node.is_leaf():
-                return
-            run_tree(node.get_down())
-            run_tree(node.get_up())
-        run_tree(self._clf._tree)

tests/Stree_test.py

@@ -1,223 +0,0 @@
-import csv
-import os
-import unittest
-
-import numpy as np
-from sklearn.datasets import make_classification
-
-from trees.Stree import Stree, Snode
-
-
-class Stree_test(unittest.TestCase):
-
-    def __init__(self, *args, **kwargs):
-        os.environ['TESTING'] = '1'
-        self._random_state = 1
-        self._clf = Stree(random_state=self._random_state,
-                            use_predictions=False)
-        self._clf.fit(*self._get_Xy())
-        super(Stree_test, self).__init__(*args, **kwargs)
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            os.environ.pop('TESTING')
-        except:
-            pass
-        
-    def _get_Xy(self):
-        X, y = make_classification(n_samples=1500, n_features=3, n_informative=3,
-                                   n_redundant=0, n_repeated=0, n_classes=2, n_clusters_per_class=2,
-                                   class_sep=1.5, flip_y=0, weights=[0.5, 0.5], random_state=self._random_state)
-        return X, y
-
-    def _check_tree(self, node: Snode):
-        """Check recursively that the nodes that are not leaves have the correct 
-        number of labels and its sons have the right number of elements in their dataset
-
-        Arguments:
-            node {Snode} -- node to check
-        """
-        if node.is_leaf():
-            return
-        y_prediction = node._clf.predict(node._X)
-        y_down = node.get_down()._y
-        y_up = node.get_up()._y
-        # Is a correct partition in terms of cadinality?
-        # i.e. The partition algorithm didn't forget any sample
-        self.assertEqual(node._y.shape[0], y_down.shape[0] + y_up.shape[0])
-        unique_y, count_y = np.unique(node._y, return_counts=True)
-        _, count_d = np.unique(y_down, return_counts=True)
-        _, count_u = np.unique(y_up, return_counts=True)
-        #
-        for i in unique_y:
-            try:
-                number_down = count_d[i]
-            except:
-                number_down = 0
-            try:
-                number_up = count_u[i]
-            except:
-                number_up = 0
-            self.assertEqual(count_y[i], number_down + number_up)
-        # Is the partition made the same as the prediction?
-        # as the node is not a leaf...
-        _, count_yp = np.unique(y_prediction, return_counts=True)
-        self.assertEqual(count_yp[0], y_up.shape[0])
-        self.assertEqual(count_yp[1], y_down.shape[0])
-        self._check_tree(node.get_down())
-        self._check_tree(node.get_up())
-
-    def test_build_tree(self):
-        """Check if the tree is built the same way as predictions of models
-        """
-        self._check_tree(self._clf._tree)
-
-    def _get_file_data(self, file_name: str) -> tuple:
-        """Return X, y from data, y is the last column in array
-
-        Arguments:
-            file_name {str} -- the file name
-
-        Returns:
-            tuple -- tuple with samples, categories
-        """
-        data = np.genfromtxt(file_name, delimiter=',')
-        data = np.array(data)
-        column_y = data.shape[1] - 1
-        fy = data[:, column_y]
-        fx = np.delete(data, column_y, axis=1)
-        return fx, fy
-
-    def _find_out(self, px: np.array, x_original: np.array, y_original) -> list:
-        """Find the original values of y for a given array of samples
-
-        Arguments:
-            px {np.array} -- array of samples to search for
-            x_original {np.array} -- original dataset
-            y_original {[type]} -- original classes
-
-        Returns:
-            np.array -- classes of the given samples
-        """
-        res = []
-        for needle in px:
-            for row in range(x_original.shape[0]):
-                if all(x_original[row, :] == needle):
-                    res.append(y_original[row])
-        return res
-
-    def test_subdatasets(self):
-        """Check if the subdatasets files have the same labels as the original dataset
-        """
-        self._clf.save_sub_datasets()
-        with open(self._clf.get_catalog_name()) as cat_file:
-            catalog = csv.reader(cat_file, delimiter=',')
-            for row in catalog:
-                X, y = self._get_Xy()
-                x_file, y_file = self._get_file_data(row[0])
-                y_original = np.array(self._find_out(x_file, X, y), dtype=int)
-                self.assertTrue(np.array_equal(y_file, y_original))
-    
-    def test_single_prediction(self):
-        X, y = self._get_Xy()
-        yp = self._clf.predict((X[0, :].reshape(-1, X.shape[1])))
-        self.assertEqual(yp[0], y[0])
-
-    def test_multiple_prediction(self):
-        # First 27 elements the predictions are the same as the truth
-        num = 27
-        X, y = self._get_Xy()
-        yp = self._clf.predict(X[:num, :])
-        self.assertListEqual(y[:num].tolist(), yp.tolist())
-
-    def test_score(self):
-        X, y = self._get_Xy()
-        accuracy_score = self._clf.score(X, y)
-        yp = self._clf.predict(X)
-        right = (yp == y).astype(int)
-        accuracy_computed = sum(right) / len(y)
-        self.assertEqual(accuracy_score, accuracy_computed)
-        self.assertGreater(accuracy_score, 0.8)
-    
-    def test_single_predict_proba(self):
-        """Check that element 28 has a prediction different that the current label
-        """
-        # Element 28 has a different prediction than the truth
-        X, y = self._get_Xy()
-        yp = self._clf.predict_proba(X[28, :].reshape(-1, X.shape[1]))
-        self.assertEqual(0, yp[0:, 0])
-        self.assertEqual(1, y[28])
-        self.assertEqual(0.29026400766, round(yp[0, 1], 11))
-
-    def test_multiple_predict_proba(self):
-        # First 27 elements the predictions are the same as the truth
-        num = 27
-        X, y = self._get_Xy()
-        yp = self._clf.predict_proba(X[:num, :])
-        self.assertListEqual(y[:num].tolist(), yp[:, 0].tolist())
-        expected_proba = [0.88395641, 0.36746962, 0.84158767, 0.34106833, 0.14269291, 0.85193236,
-                        0.29876058, 0.7282164,  0.85958616, 0.89517877, 0.99745224, 0.18860349,
-                        0.30756427, 0.8318412,  0.18981198, 0.15564624, 0.25740655, 0.22923355,
-                        0.87365959, 0.49928689, 0.95574351, 0.28761257, 0.28906333, 0.32643692,
-                        0.29788483, 0.01657364, 0.81149083]
-        self.assertListEqual(expected_proba, np.round(yp[:, 1], decimals=8).tolist())
-
-    def build_models(self):
-        """Build and train two models, model_clf will use the sklearn classifier to
-        compute predictions and split data. model_computed will use vector of
-        coefficients to compute both predictions and splitted data
-        """
-        model_clf = Stree(random_state=self._random_state,
-                            use_predictions=True)
-        model_computed = Stree(random_state=self._random_state,
-                            use_predictions=False)
-        X, y = self._get_Xy()
-        model_clf.fit(X, y)
-        model_computed.fit(X, y)
-        return model_clf, model_computed, X, y
-
-    def test_use_model_predict(self):
-        """Check that we get the same results wether we use the estimator in nodes
-        to compute labels or we use the hyperplane and the position of samples wrt to it
-        """
-        use_clf, use_math, X, _ = self.build_models()
-        self.assertListEqual(
-            use_clf.predict(X).tolist(),
-            use_math.predict(X).tolist()
-        )
-    
-    def test_use_model_score(self):
-        use_clf, use_math, X, y = self.build_models()
-        b = use_math.score(X, y)
-        self.assertEqual(
-            use_clf.score(X, y),
-           b
-        )
-        self.assertGreater(b, .95)
-
-    def test_use_model_predict_proba(self):
-        use_clf, use_math, X, _ = self.build_models()
-        self.assertListEqual(
-            use_clf.predict_proba(X).tolist(),
-            use_math.predict_proba(X).tolist()
-        )
-
-    def test_single_vs_multiple_prediction(self):
-        """Check if predicting sample by sample gives the same result as predicting
-        all samples at once
-        """
-        X, _ = self._get_Xy()
-        # Compute prediction line by line
-        yp_line = np.array([], dtype=int)
-        for xp in X:
-            yp_line = np.append(yp_line, self._clf.predict(xp.reshape(-1, X.shape[1])))
-        # Compute prediction at once
-        yp_once = self._clf.predict(X)
-        #
-        self.assertListEqual(yp_line.tolist(), yp_once.tolist())
-
-
-
-
-

trees/Siterator.py

@@ -1,34 +0,0 @@
-'''
-__author__ = "Ricardo Montañana Gómez"
-__copyright__ = "Copyright 2020, Ricardo Montañana Gómez"
-__license__ = "MIT"
-__version__ = "0.9"
-Inorder iterator for the binary tree of Snodes
-Uses LinearSVC
-'''
-
-from trees.Snode import Snode
-
-
-class Siterator:
-    """Inorder iterator
-    """
-
-    def __init__(self, tree: Snode):
-        self._stack = []
-        self._push(tree)
-
-    def __iter__(self):
-        return self
-
-    def _push(self, node: Snode):
-        while (node is not None):
-            self._stack.insert(0, node)
-            node = node.get_down()
-
-    def __next__(self) -> Snode:
-        if len(self._stack) == 0:
-            raise StopIteration()
-        node = self._stack.pop()
-        self._push(node.get_up())
-        return node

trees/Snode.py

@@ -1,70 +0,0 @@
-'''
-__author__ = "Ricardo Montañana Gómez"
-__copyright__ = "Copyright 2020, Ricardo Montañana Gómez"
-__license__ = "MIT"
-__version__ = "0.9"
-Node of the Stree (binary tree)
-'''
-
-import os
-
-import numpy as np
-from sklearn.svm import LinearSVC
-
-
-class Snode:
-    def __init__(self, clf: LinearSVC, X: np.ndarray, y: np.ndarray, title: str):
-        self._clf = clf
-        self._vector = None if clf is None else clf.coef_
-        self._interceptor = 0. if clf is None else clf.intercept_
-        self._title = title
-        self._belief = 0.  # belief of the prediction in a leaf node based on samples
-        # Only store dataset in Testing 
-        self._X = X if os.environ.get('TESTING', 'NS') != 'NS' else None
-        self._y = y
-        self._down = None
-        self._up = None
-        self._class = None
-
-    def set_down(self, son):
-        self._down = son
-
-    def set_up(self, son):
-        self._up = son
-
-    def is_leaf(self,) -> bool:
-        return self._up is None and self._down is None
-
-    def get_down(self) -> 'Snode':
-        return self._down
-
-    def get_up(self) -> 'Snode':
-        return self._up
-
-    def make_predictor(self):
-        """Compute the class of the predictor and its belief based on the subdataset of the node
-        only if it is a leaf
-        """
-        # Clean memory
-        #self._X = None
-        #self._y = None
-        if not self.is_leaf():
-            return
-        classes, card = np.unique(self._y, return_counts=True)
-        if len(classes) > 1:
-            max_card = max(card)
-            min_card = min(card)
-            try:
-                self._belief = max_card / (max_card + min_card)
-            except:
-                self._belief = 0.
-            self._class = classes[card == max_card][0]
-        else:
-            self._belief = 1
-            self._class = classes[0]
-
-    def __str__(self) -> str:
-        if self.is_leaf():
-            return f"{self._title} - Leaf class={self._class} belief={self._belief:.6f} counts={np.unique(self._y, return_counts=True)}"
-        else:
-            return f"{self._title}"

trees/Stree.py

@@ -1,222 +0,0 @@
-'''
-__author__ = "Ricardo Montañana Gómez"
-__copyright__ = "Copyright 2020, Ricardo Montañana Gómez"
-__license__ = "MIT"
-__version__ = "0.9"
-Build an oblique tree classifier based on SVM Trees
-Uses LinearSVC
-'''
-
-import typing
-
-import numpy as np
-from sklearn.base import BaseEstimator, ClassifierMixin
-from sklearn.svm import LinearSVC
-from sklearn.utils.validation import check_X_y, check_array, check_is_fitted
-
-from trees.Snode import Snode
-from trees.Siterator import Siterator
-
-
-class Stree(BaseEstimator, ClassifierMixin):
-    """
-    """
-
-    def __init__(self, C=1.0, max_iter: int = 1000, random_state: int = 0, use_predictions: bool = False):
-        self._max_iter = max_iter
-        self._C = C
-        self._random_state = random_state
-        self._tree = None
-        self.__folder = 'data/'
-        self.__use_predictions = use_predictions
-        self.__trained = False
-        self.__proba = False
-
-    def get_params(self, deep=True):
-        """Get dict with hyperparameters and its values to accomplish sklearn rules
-        """
-        return {"C": self._C, "random_state": self._random_state, 'max_iter': self._max_iter}
-
-    def set_params(self, **parameters):
-        """Set hyperparmeters as specified by sklearn, needed in Gridsearchs
-        """
-        for parameter, value in parameters.items():
-            setattr(self, parameter, value)
-        return self
-
-    def _linear_function(self, data: np.array, node: Snode) -> np.array:
-        coef = node._vector[0, :].reshape(-1, data.shape[1])
-        return data.dot(coef.T) + node._interceptor[0]
-
-    def _split_data(self, node: Snode, data: np.ndarray, indices: np.ndarray) -> list:
-        if self.__use_predictions:
-            yp = node._clf.predict(data)
-            down = (yp == 1).reshape(-1, 1)
-            res = np.expand_dims(node._clf.decision_function(data), 1)
-        else:
-            # doesn't work with multiclass as each sample has to do inner product with its own coeficients
-            # computes positition of every sample is w.r.t. the hyperplane
-            res = self._linear_function(data, node)
-            down = res > 0
-        up = ~down
-        data_down = data[down[:, 0]] if any(down) else None
-        indices_down = indices[down[:, 0]] if any(down) else None
-        res_down = res[down[:, 0]] if any(down) else None
-        data_up = data[up[:, 0]] if any(up) else None
-        indices_up = indices[up[:, 0]] if any(up) else None
-        res_up = res[up[:, 0]] if any(up) else None
-        return [data_up, indices_up, data_down, indices_down, res_up, res_down]
-
-    def fit(self, X: np.ndarray, y: np.ndarray, title: str = 'root') -> 'Stree':
-        X, y = check_X_y(X, y.ravel())
-        self.n_features_in_ = X.shape[1]
-        self._tree = self.train(X, y.ravel(), title)
-        self._build_predictor()
-        self.__trained = True
-        return self
-
-    def _build_predictor(self):
-        """Process the leaves to make them predictors
-        """
-        def run_tree(node: Snode):
-            if node.is_leaf():
-                node.make_predictor()
-                return
-            run_tree(node.get_down())
-            run_tree(node.get_up())
-        run_tree(self._tree)
-
-    def train(self, X: np.ndarray, y: np.ndarray, title: str = 'root') -> Snode:
-        if np.unique(y).shape[0] == 1:
-            # only 1 class => pure dataset
-            return Snode(None, X, y, title + ', <pure>')
-        # Train the model
-        clf = LinearSVC(max_iter=self._max_iter, C=self._C,
-                        random_state=self._random_state)
-        clf.fit(X, y)
-        tree = Snode(clf, X, y, title)
-        X_U, y_u, X_D, y_d, _, _ = self._split_data(tree, X, y)
-        if X_U is None or X_D is None:
-            # didn't part anything
-            return Snode(clf, X, y, title + ', <cgaf>')
-        tree.set_up(self.train(X_U, y_u, title + ' - Up'))
-        tree.set_down(self.train(X_D, y_d, title + ' - Down'))
-        return tree
-
-    def _reorder_results(self, y: np.array, indices: np.array) -> np.array:
-        y_ordered = np.zeros(y.shape, dtype=int if y.ndim == 1 else float)
-        indices = indices.astype(int)
-        for i, index in enumerate(indices):
-            y_ordered[index] = y[i]
-        return y_ordered
-
-    def predict(self, X: np.array) -> np.array:
-        def predict_class(xp: np.array, indices: np.array, node: Snode) -> np.array:
-            if xp is None:
-                return [], []
-            if node.is_leaf():
-                # set a class for every sample in dataset
-                prediction = np.full((xp.shape[0], 1), node._class)
-                return prediction, indices
-            u, i_u, d, i_d, _, _ = self._split_data(node, xp, indices)
-            k, l = predict_class(d, i_d, node.get_down())
-            m, n = predict_class(u, i_u, node.get_up())
-            return np.append(k, m), np.append(l, n)
-        # sklearn check
-        check_is_fitted(self)
-        # Input validation
-        X = check_array(X)
-        # setup prediction & make it happen
-        indices = np.arange(X.shape[0])
-        return self._reorder_results(*predict_class(X, indices, self._tree))
-
-    def predict_proba(self, X: np.array) -> np.array:
-        """Computes an approximation of the probability of samples belonging to class 1 
-        (nothing more, nothing less)
-
-        :param X: dataset
-        :type X: np.array
-        """
-        def predict_class(xp: np.array, indices: np.array, dist: np.array, node: Snode) -> np.array:
-            """Run the tree to compute predictions
-
-            :param xp: subdataset of samples
-            :type xp: np.array
-            :param indices: indices of subdataset samples to rebuild original order
-            :type indices: np.array
-            :param dist: distances of every sample to the hyperplane or the father node
-            :type dist: np.array
-            :param node: node of the leaf with the class
-            :type node: Snode
-            :return: array of labels and distances, array of indices
-            :rtype: np.array
-            """
-            if xp is None:
-                return [], []
-            if node.is_leaf():
-                # set a class for every sample in dataset
-                prediction = np.full((xp.shape[0], 1), node._class)
-                prediction_proba = dist
-                return np.append(prediction, prediction_proba, axis=1), indices
-            u, i_u, d, i_d, r_u, r_d = self._split_data(node, xp, indices)
-            k, l = predict_class(d, i_d, r_d, node.get_down())
-            m, n = predict_class(u, i_u, r_u, node.get_up())
-            return np.append(k, m), np.append(l, n)
-        # sklearn check
-        check_is_fitted(self)
-        # Input validation
-        X = check_array(X)
-        # setup prediction & make it happen
-        indices = np.arange(X.shape[0])
-        result, indices = predict_class(X, indices, [], self._tree)
-        result = result.reshape(X.shape[0], 2)
-        # Turn distances to hyperplane into probabilities based on fitting distances
-        # of samples to its hyperplane that classified them, to the sigmoid function
-        result[:, 1] = 1 / (1 + np.exp(-result[:, 1]))
-        return self._reorder_results(result, indices)
-
-    def score(self, X: np.array, y: np.array) -> float:
-        """Return accuracy
-        """
-        if not self.__trained:
-            self.fit(X, y)
-        yp = self.predict(X).reshape(y.shape)
-        right = (yp == y).astype(int)
-        return np.sum(right) / len(y)
-
-    def __iter__(self):
-        return Siterator(self._tree)
-
-    def __str__(self) -> str:
-        output = ''
-        for i in self:
-            output += str(i) + '\n'
-        return output
-
-    def _save_datasets(self, tree: Snode, catalog: typing.TextIO, number: int):
-        """Save the dataset of the node in a csv file
-
-        :param tree: node with data to save
-        :type tree: Snode
-        :param catalog: catalog file handler
-        :type catalog: typing.TextIO
-        :param number: sequential number for the generated file name
-        :type number: int
-        """
-        data = np.append(tree._X, tree._y.reshape(-1, 1), axis=1)
-        name = f"{self.__folder}dataset{number}.csv"
-        np.savetxt(name, data, delimiter=",")
-        catalog.write(f"{name}, - {str(tree)}")
-        if tree.is_leaf():
-            return
-        self._save_datasets(tree.get_down(), catalog, number + 1)
-        self._save_datasets(tree.get_up(), catalog, number + 2)
-
-    def get_catalog_name(self):
-        return self.__folder + "catalog.txt"
-
-    def save_sub_datasets(self):
-        """Save the every dataset stored in the tree to check with manual classifier
-        """
-        with open(self.get_catalog_name(), 'w', encoding='utf-8') as catalog:
-            self._save_datasets(self._tree, catalog, 1)