Add python 3.10 to CI

* Add graphviz representation of the tree

* Complete graphviz test
Add comments to some tests

* Add optional title to tree graph

* Add fontcolor keyword to nodes of the tree

* Add color keyword to arrows of graph

* Update version file to 1.2.4

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,56 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '16 17 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/main.yml

@@ -12,8 +12,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest]
+        os: [macos-latest, ubuntu-latest, windows-latest]
-        python: [3.8]
+        python: [3.8, "3.10"]
 
     steps:
       - uses: actions/checkout@v2

CITATION.cff

@@ -0,0 +1,37 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: "Montañana"
+    given-names: "Ricardo"
+    orcid: "https://orcid.org/0000-0003-3242-5452"
+  - family-names: "Gámez"
+    given-names: "José A."
+    orcid: "https://orcid.org/0000-0003-1188-1117"
+  - family-names: "Puerta"
+    given-names: "José M."
+    orcid: "https://orcid.org/0000-0002-9164-5191"
+title: "STree"
+version: 1.2.3
+doi: 10.5281/zenodo.5504083
+date-released: 2021-11-02
+url: "https://github.com/Doctorado-ML/STree"
+preferred-citation:
+  type: article
+  authors:
+    - family-names: "Montañana"
+      given-names: "Ricardo"
+      orcid: "https://orcid.org/0000-0003-3242-5452"
+    - family-names: "Gámez"
+      given-names: "José A."
+      orcid: "https://orcid.org/0000-0003-1188-1117"
+    - family-names: "Puerta"
+      given-names: "José M."
+      orcid: "https://orcid.org/0000-0002-9164-5191"
+  doi: "10.1007/978-3-030-85713-4_6"
+  journal: "Lecture Notes in Computer Science"
+  month: 9
+  start: 54
+  end: 64
+  title: "STree: A Single Multi-class Oblique Decision Tree Based on Support Vector Machines"
+  volume: 12882
+  year: 2021

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 Doctorado-ML
+Copyright (c) 2020-2021, Ricardo Montañana Gómez
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,56 @@
+SHELL := /bin/bash
+.DEFAULT_GOAL := help
+.PHONY: coverage deps help lint push test doc build
+
+coverage:  ## Run tests with coverage
+	coverage erase
+	coverage run -m unittest -v stree.tests
+	coverage report -m
+
+deps:  ## Install dependencies
+	pip install -r requirements.txt
+
+devdeps:  ## Install development dependencies
+	pip install black pip-audit flake8 mypy coverage
+
+lint:  ## Lint and static-check
+	black stree
+	flake8 stree
+	mypy stree
+
+push:  ## Push code with tags
+	git push && git push --tags
+
+test:  ## Run tests
+	python -m unittest -v stree.tests
+
+doc:  ## Update documentation
+	make -C docs --makefile=Makefile html
+
+build:  ## Build package
+	rm -fr dist/*
+	rm -fr build/*
+	python setup.py sdist bdist_wheel
+
+doc-clean:  ## Update documentation
+	make -C docs --makefile=Makefile clean
+
+audit: ## Audit pip
+	pip-audit
+
+help: ## Show help message
+	@IFS=$$'\n' ; \
+	help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##/:/'`); \
+	printf "%s\n\n" "Usage: make [task]"; \
+	printf "%-20s %s\n" "task" "help" ; \
+	printf "%-20s %s\n" "------" "----" ; \
+	for help_line in $${help_lines[@]}; do \
+		IFS=$$':' ; \
+		help_split=($$help_line) ; \
+		help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
+		help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \
+		printf '\033[36m'; \
+		printf "%-20s %s" $$help_command ; \
+		printf '\033[0m'; \
+		printf "%s\n" $$help_info; \
+	done

README.md

@@ -1,8 +1,12 @@
 ![CI](https://github.com/Doctorado-ML/STree/workflows/CI/badge.svg)
 [![codecov](https://codecov.io/gh/doctorado-ml/stree/branch/master/graph/badge.svg)](https://codecov.io/gh/doctorado-ml/stree)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/35fa3dfd53a24a339344b33d9f9f2f3d)](https://www.codacy.com/gh/Doctorado-ML/STree?utm_source=github.com&utm_medium=referral&utm_content=Doctorado-ML/STree&utm_campaign=Badge_Grade)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Doctorado-ML/STree.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Doctorado-ML/STree/context:python)
+[![PyPI version](https://badge.fury.io/py/STree.svg)](https://badge.fury.io/py/STree)
+![https://img.shields.io/badge/python-3.8%2B-blue](https://img.shields.io/badge/python-3.8%2B-brightgreen)
+[![DOI](https://zenodo.org/badge/262658230.svg)](https://zenodo.org/badge/latestdoi/262658230)
 
-# Stree
+# STree
 
 Oblique Tree classifier based on SVM nodes. The nodes are built and splitted with sklearn SVC models. Stree is a sklearn estimator and can be integrated in pipelines, grid searches, etc.
 
@@ -14,39 +18,41 @@ Oblique Tree classifier based on SVM nodes. The nodes are built and splitted wit
 pip install git+https://github.com/doctorado-ml/stree
 ```
 
+## Documentation
+
+Can be found in [stree.readthedocs.io](https://stree.readthedocs.io/en/stable/)
+
 ## Examples
 
 ### Jupyter notebooks
 
-- [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Doctorado-ML/STree/master?urlpath=lab/tree/notebooks/benchmark.ipynb) Benchmark
+- [![benchmark](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/benchmark.ipynb) Benchmark
 
-- [![Test](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/benchmark.ipynb) Benchmark
+- [![features](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/features.ipynb) Some features
-
-- [![Test2](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/features.ipynb) Test features
-
-- [![Adaboost](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/adaboost.ipynb) Adaboost
 
 - [![Gridsearch](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/gridsearch.ipynb) Gridsearch
 
-- [![Test Graphics](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/test_graphs.ipynb) Test Graphics
+- [![Ensemble](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/ensemble.ipynb) Ensembles
 
 ## Hyperparameters
 
 |     | **Hyperparameter**  | **Type/Values**                                                | **Default** | **Meaning**                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| --- | ------------------ | ------------------------------------------------------ | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| --- | ------------------- | -------------------------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | \*  | C                   | \<float\>                                                      | 1.0         | Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| \*  | kernel             | {"linear", "poly", "rbf"}                              | linear      | Specifies the kernel type to be used in the algorithm. It must be one of ‘linear’, ‘poly’ or ‘rbf’.                                                                                                                                                                                                                                                                                                                                                  |
+| \*  | kernel              | {"liblinear", "linear", "poly", "rbf", "sigmoid"}              | linear      | Specifies the kernel type to be used in the algorithm. It must be one of ‘liblinear’, ‘linear’, ‘poly’ or ‘rbf’. liblinear uses [liblinear](https://www.csie.ntu.edu.tw/~cjlin/liblinear/) library and the rest uses [libsvm](https://www.csie.ntu.edu.tw/~cjlin/libsvm/) library through scikit-learn library                                                                                                                                                                                                                                                                                                                          |
 | \*  | max_iter            | \<int\>                                                        | 1e5         | Hard limit on iterations within solver, or -1 for no limit.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | \*  | random_state        | \<int\>                                                        | None        | Controls the pseudo random number generation for shuffling the data for probability estimates. Ignored when probability is False.<br>Pass an int for reproducible output across multiple function calls                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 |     | max_depth           | \<int\>                                                        | None        | Specifies the maximum depth of the tree                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 | \*  | tol                 | \<float\>                                                      | 1e-4        | Tolerance for stopping criterion.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | \*  | degree              | \<int\>                                                        | 3           | Degree of the polynomial kernel function (‘poly’). Ignored by all other kernels.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| \*  | gamma              | {"scale", "auto"} or \<float\>                         | scale       | Kernel coefficient for ‘rbf’ and ‘poly’.<br>if gamma='scale' (default) is passed then it uses 1 / (n_features \* X.var()) as value of gamma,<br>if ‘auto’, uses 1 / n_features.                                                                                                                                                                                                                                                                      |
+| \*  | gamma               | {"scale", "auto"} or \<float\>                                 | scale       | Kernel coefficient for ‘rbf’, ‘poly’ and ‘sigmoid’.<br>if gamma='scale' (default) is passed then it uses 1 / (n_features \* X.var()) as value of gamma,<br>if ‘auto’, uses 1 / n_features.                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-|     | split_criteria     | {"impurity", "max_samples"}                            | impurity    | Decides (just in case of a multi class classification) which column (class) use to split the dataset in a node\*\*                                                                                                                                                                                                                                                                                                                                   |
+|     | split_criteria      | {"impurity", "max_samples"}                                    | impurity    | Decides (just in case of a multi class classification) which column (class) use to split the dataset in a node\*\*. max_samples is incompatible with 'ovo' multiclass_strategy                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
 |     | criterion           | {“gini”, “entropy”}                                            | entropy     | The function to measure the quality of a split (only used if max_features != num_features). <br>Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain.                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 |     | min_samples_split   | \<int\>                                                        | 0           | The minimum number of samples required to split an internal node. 0 (default) for any                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 |     | max_features        | \<int\>, \<float\> <br><br>or {“auto”, “sqrt”, “log2”}         | None        | The number of features to consider when looking for the split:<br>If int, then consider max_features features at each split.<br>If float, then max_features is a fraction and int(max_features \* n_features) features are considered at each split.<br>If “auto”, then max_features=sqrt(n_features).<br>If “sqrt”, then max_features=sqrt(n_features).<br>If “log2”, then max_features=log2(n_features).<br>If None, then max_features=n_features.                                                                                                                                                                                    |
-|     | splitter           | {"best", "random"}                                     | random      | The strategy used to choose the feature set at each node (only used if max_features != num_features). <br>Supported strategies are “best” to choose the best feature set and “random” to choose a random combination. <br>The algorithm generates 5 candidates at most to choose from in both strategies.                                                                                                                                            |
+|     | splitter            | {"best", "random", "trandom", "mutual", "cfs", "fcbf", "iwss"} | "random"    | The strategy used to choose the feature set at each node (only used if max_features < num_features). Supported strategies are: **“best”**: sklearn SelectKBest algorithm is used in every node to choose the max_features best features. **“random”**: The algorithm generates 5 candidates and choose the best (max. info. gain) of them. **“trandom”**: The algorithm generates only one random combination. **"mutual"**: Chooses the best features w.r.t. their mutual info with the label. **"cfs"**: Apply Correlation-based Feature Selection. **"fcbf"**: Apply Fast Correlation-Based Filter. **"iwss"**: IWSS based algorithm |
+|     | normalize           | \<bool\>                                                       | False       | If standardization of features should be applied on each node with the samples that reach it                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| \*  | multiclass_strategy | {"ovo", "ovr"}                                                 | "ovo"       | Strategy to use with multiclass datasets, **"ovo"**: one versus one. **"ovr"**: one versus rest                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
 
 \* Hyperparameter used by the support vector classifier of every node
 
@@ -63,3 +69,11 @@ Once we have the column to take into account for the split, the algorithm splits
 ```bash
 python -m unittest -v stree.tests
 ```
+
+## License
+
+STree is [MIT](https://github.com/doctorado-ml/stree/blob/master/LICENSE) licensed
+
+## Reference
+
+R. Montañana, J. A. Gámez, J. M. Puerta, "STree: a single multi-class oblique decision tree based on support vector machines.", 2021 LNAI 12882, pg. 54-64

codecov.yml

@@ -1,8 +1,8 @@
-overage:
+coverage:
   status:
     project:
       default:
-        target: 90%
+        target: 100%
 comment:
   layout: "reach, diff, flags, files"
   behavior: default

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+sphinx-rtd-theme
+myst-parser
+mufs

docs/source/api/Siterator.rst

@@ -0,0 +1,9 @@
+Siterator
+=========
+
+.. automodule:: Splitter
+.. autoclass:: Siterator
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

docs/source/api/Snode.rst

@@ -0,0 +1,9 @@
+Snode
+=====
+
+.. automodule:: Splitter
+.. autoclass:: Snode
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

docs/source/api/Splitter.rst

@@ -0,0 +1,9 @@
+Splitter
+========
+
+.. automodule:: Splitter
+.. autoclass:: Splitter
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

docs/source/api/Stree.rst

@@ -0,0 +1,9 @@
+Stree
+=====
+
+.. automodule:: stree
+.. autoclass:: Stree
+    :members:
+    :undoc-members:
+    :private-members:
+    :show-inheritance:

docs/source/api/index.rst

@@ -0,0 +1,11 @@
+API index
+=========
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   Stree
+   Siterator
+   Snode
+   Splitter

docs/source/conf.py

@@ -0,0 +1,57 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+import stree
+
+sys.path.insert(0, os.path.abspath("../../stree/"))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "STree"
+copyright = "2020 - 2021, Ricardo Montañana Gómez"
+author = "Ricardo Montañana Gómez"
+
+# The full version, including alpha/beta/rc tags
+version = stree.__version__
+release = version
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ["myst_parser", "sphinx.ext.autodoc", "sphinx.ext.viewcode"]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []

docs/source/example.md

@@ -0,0 +1,42 @@
+# Examples
+
+## Notebooks
+
+- [![benchmark](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/benchmark.ipynb) Benchmark
+
+- [![features](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/features.ipynb) Some features
+
+- [![Gridsearch](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/gridsearch.ipynb) Gridsearch
+
+- [![Ensemble](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Doctorado-ML/STree/blob/master/notebooks/ensemble.ipynb) Ensembles
+
+## Sample Code
+
+```python
+import time
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_iris
+from stree import Stree
+
+random_state = 1
+X, y = load_iris(return_X_y=True)
+Xtrain, Xtest, ytrain, ytest = train_test_split(
+    X, y, test_size=0.2, random_state=random_state
+)
+now = time.time()
+print("Predicting with max_features=sqrt(n_features)")
+clf = Stree(random_state=random_state, max_features="auto")
+clf.fit(Xtrain, ytrain)
+print(f"Took {time.time() - now:.2f} seconds to train")
+print(clf)
+print(f"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}")
+print(f"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}")
+print("=" * 40)
+print("Predicting with max_features=n_features")
+clf = Stree(random_state=random_state)
+clf.fit(Xtrain, ytrain)
+print(f"Took {time.time() - now:.2f} seconds to train")
+print(clf)
+print(f"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}")
+print(f"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}")
+```

docs/source/hyperparameters.md

@@ -0,0 +1,29 @@
+# Hyperparameters
+
+|     | **Hyperparameter**  | **Type/Values**                                                | **Default** | **Meaning**                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| --- | ------------------- | -------------------------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| \*  | C                   | \<float\>                                                      | 1.0         | Regularization parameter. The strength of the regularization is inversely proportional to C. Must be strictly positive.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| \*  | kernel              | {"liblinear", "linear", "poly", "rbf", "sigmoid"}              | linear      | Specifies the kernel type to be used in the algorithm. It must be one of ‘liblinear’, ‘linear’, ‘poly’ or ‘rbf’. liblinear uses [liblinear](https://www.csie.ntu.edu.tw/~cjlin/liblinear/) library and the rest uses [libsvm](https://www.csie.ntu.edu.tw/~cjlin/libsvm/) library through scikit-learn library                                                                                                                                                                                                                                                                                                                          |
+| \*  | max_iter            | \<int\>                                                        | 1e5         | Hard limit on iterations within solver, or -1 for no limit.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| \*  | random_state        | \<int\>                                                        | None        | Controls the pseudo random number generation for shuffling the data for probability estimates. Ignored when probability is False.<br>Pass an int for reproducible output across multiple function calls                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+|     | max_depth           | \<int\>                                                        | None        | Specifies the maximum depth of the tree                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| \*  | tol                 | \<float\>                                                      | 1e-4        | Tolerance for stopping criterion.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| \*  | degree              | \<int\>                                                        | 3           | Degree of the polynomial kernel function (‘poly’). Ignored by all other kernels.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| \*  | gamma               | {"scale", "auto"} or \<float\>                                 | scale       | Kernel coefficient for ‘rbf’, ‘poly’ and ‘sigmoid’.<br>if gamma='scale' (default) is passed then it uses 1 / (n_features \* X.var()) as value of gamma,<br>if ‘auto’, uses 1 / n_features.                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+|     | split_criteria      | {"impurity", "max_samples"}                                    | impurity    | Decides (just in case of a multi class classification) which column (class) use to split the dataset in a node\*\*. max_samples is incompatible with 'ovo' multiclass_strategy                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+|     | criterion           | {“gini”, “entropy”}                                            | entropy     | The function to measure the quality of a split (only used if max_features != num_features). <br>Supported criteria are “gini” for the Gini impurity and “entropy” for the information gain.                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+|     | min_samples_split   | \<int\>                                                        | 0           | The minimum number of samples required to split an internal node. 0 (default) for any                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+|     | max_features        | \<int\>, \<float\> <br><br>or {“auto”, “sqrt”, “log2”}         | None        | The number of features to consider when looking for the split:<br>If int, then consider max_features features at each split.<br>If float, then max_features is a fraction and int(max_features \* n_features) features are considered at each split.<br>If “auto”, then max_features=sqrt(n_features).<br>If “sqrt”, then max_features=sqrt(n_features).<br>If “log2”, then max_features=log2(n_features).<br>If None, then max_features=n_features.                                                                                                                                                                                    |
+|     | splitter            | {"best", "random", "trandom", "mutual", "cfs", "fcbf", "iwss"} | "random"    | The strategy used to choose the feature set at each node (only used if max_features < num_features). Supported strategies are: **“best”**: sklearn SelectKBest algorithm is used in every node to choose the max_features best features. **“random”**: The algorithm generates 5 candidates and choose the best (max. info. gain) of them. **“trandom”**: The algorithm generates only one random combination. **"mutual"**: Chooses the best features w.r.t. their mutual info with the label. **"cfs"**: Apply Correlation-based Feature Selection. **"fcbf"**: Apply Fast Correlation-Based Filter. **"iwss"**: IWSS based algorithm |
+|     | normalize           | \<bool\>                                                       | False       | If standardization of features should be applied on each node with the samples that reach it                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| \*  | multiclass_strategy | {"ovo", "ovr"}                                                 | "ovo"       | Strategy to use with multiclass datasets, **"ovo"**: one versus one. **"ovr"**: one versus rest                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+
+\* Hyperparameter used by the support vector classifier of every node
+
+\*\* **Splitting in a STree node**
+
+The decision function is applied to the dataset and distances from samples to hyperplanes are computed in a matrix. This matrix has as many columns as classes the samples belongs to (if more than two, i.e. multiclass classification) or 1 column if it's a binary class dataset. In binary classification only one hyperplane is computed and therefore only one column is needed to store the distances of the samples to it. If three or more classes are present in the dataset we need as many hyperplanes as classes are there, and therefore one column per hyperplane is needed.
+
+In case of multiclass classification we have to decide which column take into account to make the split, that depends on hyperparameter _split_criteria_, if "impurity" is chosen then STree computes information gain of every split candidate using each column and chooses the one that maximize the information gain, otherwise STree choses the column with more samples with a predicted class (the column with more positive numbers in it).
+
+Once we have the column to take into account for the split, the algorithm splits samples with positive distances to hyperplane from the rest.

docs/source/index.rst

@@ -0,0 +1,15 @@
+Welcome to STree's documentation!
+=================================
+
+.. toctree::
+   :caption: Contents:
+   :titlesonly:
+
+
+   stree
+   install
+   hyperparameters
+   example
+   api/index
+
+* :ref:`genindex`

docs/source/install.rst

@@ -0,0 +1,16 @@
+Install
+=======
+
+The main stable release
+
+``pip install stree``
+
+or the last development branch
+
+``pip install git+https://github.com/doctorado-ml/stree``
+
+Tests
+*****
+
+
+``python -m unittest -v stree.tests``

docs/source/stree.md

@@ -0,0 +1,17 @@
+# STree
+
+![CI](https://github.com/Doctorado-ML/STree/workflows/CI/badge.svg)
+[![codecov](https://codecov.io/gh/doctorado-ml/stree/branch/master/graph/badge.svg)](https://codecov.io/gh/doctorado-ml/stree)
+[![Codacy Badge](https://app.codacy.com/project/badge/Grade/35fa3dfd53a24a339344b33d9f9f2f3d)](https://www.codacy.com/gh/Doctorado-ML/STree?utm_source=github.com&utm_medium=referral&utm_content=Doctorado-ML/STree&utm_campaign=Badge_Grade)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/Doctorado-ML/STree.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Doctorado-ML/STree/context:python)
+[![PyPI version](https://badge.fury.io/py/STree.svg)](https://badge.fury.io/py/STree)
+![https://img.shields.io/badge/python-3.8%2B-blue](https://img.shields.io/badge/python-3.8%2B-brightgreen)
+[![DOI](https://zenodo.org/badge/262658230.svg)](https://zenodo.org/badge/latestdoi/262658230)
+
+Oblique Tree classifier based on SVM nodes. The nodes are built and splitted with sklearn SVC models. Stree is a sklearn estimator and can be integrated in pipelines, grid searches, etc.
+
+![Stree](./example.png)
+
+## License
+
+STree is [MIT](https://github.com/doctorado-ml/stree/blob/master/LICENSE) licensed

main.py

@@ -1,29 +0,0 @@
-import time
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_iris
-from stree import Stree
-
-random_state = 1
-
-X, y = load_iris(return_X_y=True)
-
-Xtrain, Xtest, ytrain, ytest = train_test_split(
-    X, y, test_size=0.3, random_state=random_state
-)
-
-now = time.time()
-print("Predicting with max_features=sqrt(n_features)")
-clf = Stree(C=0.01, random_state=random_state, max_features="auto")
-clf.fit(Xtrain, ytrain)
-print(f"Took {time.time() - now:.2f} seconds to train")
-print(clf)
-print(f"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}")
-print(f"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}")
-print("=" * 40)
-print("Predicting with max_features=n_features")
-clf = Stree(C=0.01, random_state=random_state)
-clf.fit(Xtrain, ytrain)
-print(f"Took {time.time() - now:.2f} seconds to train")
-print(clf)
-print(f"Classifier's accuracy (train): {clf.score(Xtrain, ytrain):.4f}")
-print(f"Classifier's accuracy (test) : {clf.score(Xtest, ytest):.4f}")

notebooks/benchmark.ipynb

@@ -17,23 +17,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#\n",
     "# Google Colab setup\n",
     "#\n",
-    "#!pip install git+https://github.com/doctorado-ml/stree"
+    "#!pip install git+https://github.com/doctorado-ml/stree\n",
+    "!pip install pandas"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "import datetime, time\n",
+    "import os\n",
     "import numpy as np\n",
     "import pandas as pd\n",
     "from sklearn.model_selection import train_test_split\n",
@@ -47,11 +49,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
     "if not os.path.isfile('data/creditcard.csv'):\n",
     "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
     "    !tar xzf creditcard.tgz"
@@ -66,19 +67,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "2021-01-14 11:30:51\n"
-     ]
-    }
-   ],
    "source": [
     "print(datetime.date.today(), time.strftime(\"%H:%M:%S\"))"
    ]
@@ -92,7 +85,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -104,20 +97,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Fraud: 0.173% 492\n",
-      "Valid: 99.827% 284,315\n"
-     ]
-    }
-   ],
    "source": [
     "print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
     "print(\"Valid: {0:.3f}% {1:,}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))"
@@ -125,7 +109,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -137,20 +121,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "X shape: (284807, 29)\n",
-      "y shape: (284807,)\n"
-     ]
-    }
-   ],
    "source": [
     "# Remove unneeded features\n",
     "y = df.Class.values\n",
@@ -167,7 +142,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -178,7 +153,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -188,7 +163,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -198,17 +173,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Stree\n",
-    "stree = Stree(random_state=random_state, C=.01, max_iter=1e3)"
+    "stree = Stree(random_state=random_state, C=.01, max_iter=1e3, kernel=\"liblinear\", multiclass_strategy=\"ovr\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -218,7 +193,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -235,7 +210,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -260,194 +235,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "************************** Linear Tree **********************\n",
-      "Train Model Linear Tree took: 10.25 seconds\n",
-      "=========== Linear Tree - Train 199,364 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   1.000000  1.000000  1.000000    199020\n",
-      "           1   1.000000  1.000000  1.000000       344\n",
-      "\n",
-      "    accuracy                       1.000000    199364\n",
-      "   macro avg   1.000000  1.000000  1.000000    199364\n",
-      "weighted avg   1.000000  1.000000  1.000000    199364\n",
-      "\n",
-      "=========== Linear Tree - Test 85,443 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999578  0.999613  0.999596     85295\n",
-      "           1   0.772414  0.756757  0.764505       148\n",
-      "\n",
-      "    accuracy                       0.999192     85443\n",
-      "   macro avg   0.885996  0.878185  0.882050     85443\n",
-      "weighted avg   0.999184  0.999192  0.999188     85443\n",
-      "\n",
-      "Confusion Matrix in Train\n",
-      "[[199020      0]\n",
-      " [     0    344]]\n",
-      "Confusion Matrix in Test\n",
-      "[[85262    33]\n",
-      " [   36   112]]\n",
-      "************************** Naive Bayes **********************\n",
-      "Train Model Naive Bayes took: 0.09943 seconds\n",
-      "=========== Naive Bayes - Train 199,364 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999692  0.978238  0.988849    199020\n",
-      "           1   0.061538  0.825581  0.114539       344\n",
-      "\n",
-      "    accuracy                       0.977975    199364\n",
-      "   macro avg   0.530615  0.901910  0.551694    199364\n",
-      "weighted avg   0.998073  0.977975  0.987340    199364\n",
-      "\n",
-      "=========== Naive Bayes - Test 85,443 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999712  0.977994  0.988734     85295\n",
-      "           1   0.061969  0.837838  0.115403       148\n",
-      "\n",
-      "    accuracy                       0.977751     85443\n",
-      "   macro avg   0.530841  0.907916  0.552068     85443\n",
-      "weighted avg   0.998088  0.977751  0.987221     85443\n",
-      "\n",
-      "Confusion Matrix in Train\n",
-      "[[194689   4331]\n",
-      " [    60    284]]\n",
-      "Confusion Matrix in Test\n",
-      "[[83418  1877]\n",
-      " [   24   124]]\n",
-      "************************** Stree (SVM Tree) **********************\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
-      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
-      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Train Model Stree (SVM Tree) took: 28.47 seconds\n",
-      "=========== Stree (SVM Tree) - Train 199,364 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999623  0.999864  0.999744    199020\n",
-      "           1   0.908784  0.781977  0.840625       344\n",
-      "\n",
-      "    accuracy                       0.999488    199364\n",
-      "   macro avg   0.954204  0.890921  0.920184    199364\n",
-      "weighted avg   0.999467  0.999488  0.999469    199364\n",
-      "\n",
-      "=========== Stree (SVM Tree) - Test 85,443 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999637  0.999918  0.999777     85295\n",
-      "           1   0.943548  0.790541  0.860294       148\n",
-      "\n",
-      "    accuracy                       0.999555     85443\n",
-      "   macro avg   0.971593  0.895229  0.930036     85443\n",
-      "weighted avg   0.999540  0.999555  0.999536     85443\n",
-      "\n",
-      "Confusion Matrix in Train\n",
-      "[[198993     27]\n",
-      " [    75    269]]\n",
-      "Confusion Matrix in Test\n",
-      "[[85288     7]\n",
-      " [   31   117]]\n",
-      "************************** Neural Network **********************\n",
-      "Train Model Neural Network took: 9.76 seconds\n",
-      "=========== Neural Network - Train 199,364 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999247  0.999844  0.999545    199020\n",
-      "           1   0.862222  0.563953  0.681898       344\n",
-      "\n",
-      "    accuracy                       0.999092    199364\n",
-      "   macro avg   0.930734  0.781899  0.840722    199364\n",
-      "weighted avg   0.999010  0.999092  0.998997    199364\n",
-      "\n",
-      "=========== Neural Network - Test 85,443 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999356  0.999871  0.999613     85295\n",
-      "           1   0.894231  0.628378  0.738095       148\n",
-      "\n",
-      "    accuracy                       0.999228     85443\n",
-      "   macro avg   0.946793  0.814125  0.868854     85443\n",
-      "weighted avg   0.999173  0.999228  0.999160     85443\n",
-      "\n",
-      "Confusion Matrix in Train\n",
-      "[[198989     31]\n",
-      " [   150    194]]\n",
-      "Confusion Matrix in Test\n",
-      "[[85284    11]\n",
-      " [   55    93]]\n",
-      "************************** SVC (linear) **********************\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/rmontanana/.virtualenvs/general/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
-      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Train Model SVC (linear) took: 8.207 seconds\n",
-      "=========== SVC (linear) - Train 199,364 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999237  0.999859  0.999548    199020\n",
-      "           1   0.872727  0.558140  0.680851       344\n",
-      "\n",
-      "    accuracy                       0.999097    199364\n",
-      "   macro avg   0.935982  0.778999  0.840199    199364\n",
-      "weighted avg   0.999018  0.999097  0.998998    199364\n",
-      "\n",
-      "=========== SVC (linear) - Test 85,443 samples =============\n",
-      "              precision    recall  f1-score   support\n",
-      "\n",
-      "           0   0.999344  0.999894  0.999619     85295\n",
-      "           1   0.910891  0.621622  0.738956       148\n",
-      "\n",
-      "    accuracy                       0.999239     85443\n",
-      "   macro avg   0.955117  0.810758  0.869287     85443\n",
-      "weighted avg   0.999191  0.999239  0.999168     85443\n",
-      "\n",
-      "Confusion Matrix in Train\n",
-      "[[198992     28]\n",
-      " [   152    192]]\n",
-      "Confusion Matrix in Test\n",
-      "[[85286     9]\n",
-      " [   56    92]]\n"
-     ]
-    }
-   ],
    "source": [
     "# Train & Test models\n",
     "models = {\n",
-    "    'Linear Tree':linear_tree, 'Naive Bayes': naive_bayes, 'Stree (SVM Tree)': stree,  \n",
+    "    'Linear Tree':linear_tree, 'Naive Bayes': naive_bayes, 'Stree    ': stree,  \n",
     "    'Neural Network': mlp, 'SVC (linear)': svc\n",
     "}\n",
     "\n",
@@ -464,26 +260,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "**************************************************************************************************************\n",
-      "*The best f1 model is Stree (SVM Tree), with a f1 score: 0.8603 in 28.4743 seconds with 0.7 samples in train dataset\n",
-      "**************************************************************************************************************\n",
-      "Model: Linear Tree\t Time:  10.25 seconds\t f1: 0.7645\n",
-      "Model: Naive Bayes\t Time:   0.10 seconds\t f1: 0.1154\n",
-      "Model: Stree (SVM Tree)\t Time:  28.47 seconds\t f1: 0.8603\n",
-      "Model: Neural Network\t Time:   9.76 seconds\t f1: 0.7381\n",
-      "Model: SVC (linear)\t Time:   8.21 seconds\t f1: 0.739\n"
-     ]
-    }
-   ],
    "source": [
     "print(\"*\"*110)\n",
     "print(f\"*The best f1 model is {best_model}, with a f1 score: {best_f1:.4} in {best_time:.6} seconds with {train_size:,} samples in train dataset\")\n",
@@ -508,32 +289,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "data": {
-      "text/plain": [
-       "{'C': 0.01,\n",
-       " 'criterion': 'entropy',\n",
-       " 'degree': 3,\n",
-       " 'gamma': 'scale',\n",
-       " 'kernel': 'linear',\n",
-       " 'max_depth': None,\n",
-       " 'max_features': None,\n",
-       " 'max_iter': 1000.0,\n",
-       " 'min_samples_split': 0,\n",
-       " 'random_state': 2020,\n",
-       " 'split_criteria': 'impurity',\n",
-       " 'splitter': 'random',\n",
-       " 'tol': 0.0001}"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
     "stree.get_params()"
    ]
@@ -556,7 +314,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.8.2-final"
   },
   "toc": {
    "base_numbering": 1,

notebooks/ensemble.ipynb

@@ -17,38 +17,43 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#\n",
     "# Google Colab setup\n",
     "#\n",
-    "#!pip install git+https://github.com/doctorado-ml/stree"
+    "#!pip install git+https://github.com/doctorado-ml/stree\n",
+    "!pip install pandas"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "import time\n",
+    "import os\n",
+    "import random\n",
     "import warnings\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
     "from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier\n",
     "from sklearn.model_selection import train_test_split\n",
     "from sklearn.exceptions import ConvergenceWarning\n",
     "from stree import Stree\n",
+    "\n",
     "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
     "if not os.path.isfile('data/creditcard.csv'):\n",
     "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
     "    !tar xzf creditcard.tgz"
@@ -56,30 +61,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Fraud: 0.173% 492\n",
-      "Valid: 99.827% 284315\n",
-      "X.shape (100492, 28)  y.shape (100492,)\n",
-      "Fraud: 0.651% 654\n",
-      "Valid: 99.349% 99838\n"
-     ]
-    }
-   ],
    "source": [
     "random_state=1\n",
     "\n",
     "def load_creditcard(n_examples=0):\n",
-    "    import pandas as pd\n",
-    "    import numpy as np\n",
-    "    import random\n",
     "    df = pd.read_csv('data/creditcard.csv')\n",
     "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
     "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
@@ -130,21 +120,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Score Train:  0.9984504719663368\n",
-      "Score Test:  0.9983415151917209\n",
-      "Took 26.09 seconds\n"
-     ]
-    }
-   ],
    "source": [
     "now = time.time()\n",
     "clf = Stree(max_depth=3, random_state=random_state, max_iter=1e3)\n",
@@ -163,7 +143,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -174,21 +154,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Kernel: linear\tTime: 43.49 seconds\tScore Train: 0.9980098\tScore Test: 0.9980762\n",
-      "Kernel: rbf\tTime: 8.86 seconds\tScore Train: 0.9934891\tScore Test: 0.9934987\n",
-      "Kernel: poly\tTime: 41.14 seconds\tScore Train: 0.9972279\tScore Test: 0.9973133\n"
-     ]
-    }
-   ],
    "source": [
     "for kernel in ['linear', 'rbf', 'poly']:\n",
     "    now = time.time()\n",
@@ -208,7 +178,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -219,21 +189,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Kernel: linear\tTime: 187.51 seconds\tScore Train: 0.9984505\tScore Test: 0.9983083\n",
-      "Kernel: rbf\tTime: 73.65 seconds\tScore Train: 0.9993461\tScore Test: 0.9985074\n",
-      "Kernel: poly\tTime: 52.19 seconds\tScore Train: 0.9993461\tScore Test: 0.9987727\n"
-     ]
-    }
-   ],
    "source": [
     "for kernel in ['linear', 'rbf', 'poly']:\n",
     "    now = time.time()\n",
@@ -261,7 +221,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.8.2-final"
   }
  },
  "nbformat": 4,

notebooks/features.ipynb

@@ -17,24 +17,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#\n",
     "# Google Colab setup\n",
     "#\n",
-    "#!pip install git+https://github.com/doctorado-ml/stree"
+    "#!pip install git+https://github.com/doctorado-ml/stree\n",
+    "!pip install pandas"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "import time\n",
+    "import random\n",
     "import warnings\n",
+    "import os\n",
     "import numpy as np\n",
     "import pandas as pd\n",
     "from sklearn.svm import SVC\n",
@@ -42,6 +45,7 @@
     "from sklearn.utils.estimator_checks import check_estimator\n",
     "from sklearn.datasets import make_classification, load_iris, load_wine\n",
     "from sklearn.model_selection import train_test_split\n",
+    "from sklearn.utils.class_weight import compute_sample_weight\n",
     "from sklearn.exceptions import ConvergenceWarning\n",
     "from stree import Stree\n",
     "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
@@ -49,13 +53,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
    "outputs": [],
    "source": [
-    "import os\n",
     "if not os.path.isfile('data/creditcard.csv'):\n",
     "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
     "    !tar xzf creditcard.tgz"
@@ -63,31 +66,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Fraud: 0.173% 492\n",
-      "Valid: 99.827% 284315\n",
-      "X.shape (5492, 28)  y.shape (5492,)\n",
-      "Fraud: 9.086% 499\n",
-      "Valid: 90.914% 4993\n",
-      "[0.09079084 0.09079084 0.09079084 0.09079084] [0.09101942 0.09101942 0.09101942 0.09101942]\n"
-     ]
-    }
-   ],
    "source": [
     "random_state=1\n",
     "\n",
     "def load_creditcard(n_examples=0):\n",
-    "    import pandas as pd\n",
-    "    import numpy as np\n",
-    "    import random\n",
     "    df = pd.read_csv('data/creditcard.csv')\n",
     "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
     "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
@@ -119,17 +106,8 @@
     "Xtest = data[1]\n",
     "ytrain = data[2]\n",
     "ytest = data[3]\n",
-    "_, data = np.unique(ytrain, return_counts=True)\n",
+    "weights = compute_sample_weight(\"balanced\", ytrain)\n",
-    "wtrain = (data[1] / np.sum(data),  data[0] / np.sum(data))\n",
+    "weights_test = compute_sample_weight(\"balanced\", ytest)\n",
-    "_, data = np.unique(ytest, return_counts=True)\n",
-    "wtest = (data[1] / np.sum(data),  data[0] / np.sum(data))\n",
-    "# Set weights inverse to its count class in dataset\n",
-    "weights = np.ones(Xtrain.shape[0],)\n",
-    "weights[ytrain==0] = wtrain[0]\n",
-    "weights[ytrain==1] = wtrain[1]\n",
-    "weights_test = np.ones(Xtest.shape[0],)\n",
-    "weights_test[ytest==0] = wtest[0]\n",
-    "weights_test[ytest==1] = wtest[1]\n",
     "print(weights[:4], weights_test[:4])"
    ]
   },
@@ -150,22 +128,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Accuracy of Train without weights 0.9849115504682622\n",
-      "Accuracy of Train with    weights 0.9849115504682622\n",
-      "Accuracy of Tests without weights 0.9848300970873787\n",
-      "Accuracy of Tests with    weights 0.9805825242718447\n"
-     ]
-    }
-   ],
    "source": [
     "C = 23\n",
     "print(\"Accuracy of Train without weights\", Stree(C=C, random_state=1).fit(Xtrain, ytrain).score(Xtrain, ytrain))\n",
@@ -184,21 +151,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Time: 26.59s\tKernel: linear\tAccuracy_train: 0.9846514047866806\tAccuracy_test: 0.9848300970873787\n",
-      "Time: 0.56s\tKernel: rbf\tAccuracy_train: 0.9947970863683663\tAccuracy_test: 0.9866504854368932\n",
-      "Time: 0.23s\tKernel: poly\tAccuracy_train: 0.9955775234131113\tAccuracy_test: 0.9824029126213593\n"
-     ]
-    }
-   ],
    "source": [
     "random_state=1\n",
     "for kernel in ['linear', 'rbf', 'poly']:\n",
@@ -219,77 +176,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "************** C=0.001 ****************************\n",
-      "Classifier's accuracy (train): 0.9823\n",
-      "Classifier's accuracy (test) : 0.9836\n",
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down, <cgaf> - Leaf class=0 belief= 0.981455 impurity=0.1332 counts=(array([0, 1]), array([3493,   66]))\n",
-      "root - Up, <cgaf> - Leaf class=1 belief= 0.992982 impurity=0.0603 counts=(array([0, 1]), array([  2, 283]))\n",
-      "\n",
-      "**************************************************\n",
-      "************** C=0.01 ****************************\n",
-      "Classifier's accuracy (train): 0.9834\n",
-      "Classifier's accuracy (test) : 0.9842\n",
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down, <cgaf> - Leaf class=0 belief= 0.982288 impurity=0.1284 counts=(array([0, 1]), array([3494,   63]))\n",
-      "root - Up, <cgaf> - Leaf class=1 belief= 0.996516 impurity=0.0335 counts=(array([0, 1]), array([  1, 286]))\n",
-      "\n",
-      "**************************************************\n",
-      "************** C=1 ****************************\n",
-      "Classifier's accuracy (train): 0.9844\n",
-      "Classifier's accuracy (test) : 0.9848\n",
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
-      "root - Down - Down, <cgaf> - Leaf class=0 belief= 0.983108 impurity=0.1236 counts=(array([0, 1]), array([3492,   60]))\n",
-      "root - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
-      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
-      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n",
-      "\n",
-      "**************************************************\n",
-      "************** C=5 ****************************\n",
-      "Classifier's accuracy (train): 0.9847\n",
-      "Classifier's accuracy (test) : 0.9848\n",
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
-      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3492,   60]))\n",
-      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983385 impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
-      "root - Down - Down - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
-      "root - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
-      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
-      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n",
-      "\n",
-      "**************************************************\n",
-      "************** C=17 ****************************\n",
-      "Classifier's accuracy (train): 0.9847\n",
-      "Classifier's accuracy (test) : 0.9848\n",
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
-      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
-      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983380 impurity=0.1220 counts=(array([0, 1]), array([3491,   59]))\n",
-      "root - Down - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Down - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=1.0000 counts=(array([0, 1]), array([1, 1]))\n",
-      "root - Down - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Down - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
-      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
-      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
-      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n",
-      "\n",
-      "**************************************************\n",
-      "59.0161 secs\n"
-     ]
-    }
-   ],
    "source": [
     "t = time.time()\n",
     "for C in (.001, .01, 1, 5, 17):\n",
@@ -313,29 +204,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
-      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
-      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983380 impurity=0.1220 counts=(array([0, 1]), array([3491,   59]))\n",
-      "root - Down - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Down - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=1.0000 counts=(array([0, 1]), array([1, 1]))\n",
-      "root - Down - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Down - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
-      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
-      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
-      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n"
-     ]
-    }
-   ],
    "source": [
     "#check iterator\n",
     "for i in list(clf):\n",
@@ -344,29 +217,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "root feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.4391 counts=(array([0, 1]), array([3495,  349]))\n",
-      "root - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1236 counts=(array([0, 1]), array([3493,   60]))\n",
-      "root - Down - Down feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.1220 counts=(array([0, 1]), array([3492,   59]))\n",
-      "root - Down - Down - Down, <cgaf> - Leaf class=0 belief= 0.983380 impurity=0.1220 counts=(array([0, 1]), array([3491,   59]))\n",
-      "root - Down - Down - Up, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Down - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=1.0000 counts=(array([0, 1]), array([1, 1]))\n",
-      "root - Down - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([1]))\n",
-      "root - Down - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([1]))\n",
-      "root - Up feaures=(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) impurity=0.0593 counts=(array([0, 1]), array([  2, 289]))\n",
-      "root - Up - Down, <pure> - Leaf class=0 belief= 1.000000 impurity=0.0000 counts=(array([0]), array([2]))\n",
-      "root - Up - Up, <pure> - Leaf class=1 belief= 1.000000 impurity=0.0000 counts=(array([1]), array([289]))\n"
-     ]
-    }
-   ],
    "source": [
     "#check iterator again\n",
     "for i in clf:\n",
@@ -382,73 +237,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "1 functools.partial(<function check_no_attributes_set_in_init at 0x16817f670>, 'Stree')\n",
-      "2 functools.partial(<function check_estimators_dtypes at 0x168179820>, 'Stree')\n",
-      "3 functools.partial(<function check_fit_score_takes_y at 0x168179700>, 'Stree')\n",
-      "4 functools.partial(<function check_sample_weights_pandas_series at 0x168174040>, 'Stree')\n",
-      "5 functools.partial(<function check_sample_weights_not_an_array at 0x168174160>, 'Stree')\n",
-      "6 functools.partial(<function check_sample_weights_list at 0x168174280>, 'Stree')\n",
-      "7 functools.partial(<function check_sample_weights_shape at 0x1681743a0>, 'Stree')\n",
-      "8 functools.partial(<function check_sample_weights_invariance at 0x1681744c0>, 'Stree', kind='ones')\n",
-      "10 functools.partial(<function check_estimators_fit_returns_self at 0x16817b8b0>, 'Stree')\n",
-      "11 functools.partial(<function check_estimators_fit_returns_self at 0x16817b8b0>, 'Stree', readonly_memmap=True)\n",
-      "12 functools.partial(<function check_complex_data at 0x168174670>, 'Stree')\n",
-      "13 functools.partial(<function check_dtype_object at 0x1681745e0>, 'Stree')\n",
-      "14 functools.partial(<function check_estimators_empty_data_messages at 0x1681799d0>, 'Stree')\n",
-      "15 functools.partial(<function check_pipeline_consistency at 0x1681795e0>, 'Stree')\n",
-      "16 functools.partial(<function check_estimators_nan_inf at 0x168179af0>, 'Stree')\n",
-      "17 functools.partial(<function check_estimators_overwrite_params at 0x16817f550>, 'Stree')\n",
-      "18 functools.partial(<function check_estimator_sparse_data at 0x168172ee0>, 'Stree')\n",
-      "19 functools.partial(<function check_estimators_pickle at 0x168179d30>, 'Stree')\n",
-      "20 functools.partial(<function check_estimator_get_tags_default_keys at 0x168181790>, 'Stree')\n",
-      "21 functools.partial(<function check_classifier_data_not_an_array at 0x16817f8b0>, 'Stree')\n",
-      "22 functools.partial(<function check_classifiers_one_label at 0x16817b430>, 'Stree')\n",
-      "23 functools.partial(<function check_classifiers_classes at 0x16817bd30>, 'Stree')\n",
-      "24 functools.partial(<function check_estimators_partial_fit_n_features at 0x168179e50>, 'Stree')\n",
-      "25 functools.partial(<function check_classifiers_train at 0x16817b550>, 'Stree')\n",
-      "26 functools.partial(<function check_classifiers_train at 0x16817b550>, 'Stree', readonly_memmap=True)\n",
-      "27 functools.partial(<function check_classifiers_train at 0x16817b550>, 'Stree', readonly_memmap=True, X_dtype='float32')\n",
-      "28 functools.partial(<function check_classifiers_regression_target at 0x168181280>, 'Stree')\n",
-      "29 functools.partial(<function check_supervised_y_no_nan at 0x1681720d0>, 'Stree')\n",
-      "30 functools.partial(<function check_supervised_y_2d at 0x16817baf0>, 'Stree')\n",
-      "31 functools.partial(<function check_estimators_unfitted at 0x16817b9d0>, 'Stree')\n",
-      "32 functools.partial(<function check_non_transformer_estimators_n_iter at 0x16817fdc0>, 'Stree')\n",
-      "33 functools.partial(<function check_decision_proba_consistency at 0x1681813a0>, 'Stree')\n",
-      "34 functools.partial(<function check_parameters_default_constructible at 0x16817fb80>, 'Stree')\n",
-      "35 functools.partial(<function check_methods_sample_order_invariance at 0x168174d30>, 'Stree')\n",
-      "36 functools.partial(<function check_methods_subset_invariance at 0x168174c10>, 'Stree')\n",
-      "37 functools.partial(<function check_fit2d_1sample at 0x168174e50>, 'Stree')\n",
-      "38 functools.partial(<function check_fit2d_1feature at 0x168174f70>, 'Stree')\n",
-      "39 functools.partial(<function check_get_params_invariance at 0x168181040>, 'Stree')\n",
-      "40 functools.partial(<function check_set_params at 0x168181160>, 'Stree')\n",
-      "41 functools.partial(<function check_dict_unchanged at 0x168174790>, 'Stree')\n",
-      "42 functools.partial(<function check_dont_overwrite_parameters at 0x168174940>, 'Stree')\n",
-      "43 functools.partial(<function check_fit_idempotent at 0x168181550>, 'Stree')\n",
-      "44 functools.partial(<function check_n_features_in at 0x1681815e0>, 'Stree')\n",
-      "45 functools.partial(<function check_fit1d at 0x1681790d0>, 'Stree')\n",
-      "46 functools.partial(<function check_fit2d_predict1d at 0x168174a60>, 'Stree')\n",
-      "47 functools.partial(<function check_requires_y_none at 0x168181670>, 'Stree')\n"
-     ]
-    }
-   ],
    "source": [
     "# Make checks one by one\n",
     "c = 0\n",
     "checks = check_estimator(Stree(), generate_only=True)\n",
     "for check in checks:\n",
     "    c += 1\n",
-    "    if c == 9:\n",
-    "        pass\n",
-    "    else:\n",
     "    print(c, check[1])\n",
     "    check[1](check[0])"
    ]
@@ -552,7 +351,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.1"
+   "version": "3.8.2-final"
   }
  },
  "nbformat": 4,

notebooks/gridsearch.ipynb

@@ -18,19 +18,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "#\n",
     "# Google Colab setup\n",
     "#\n",
-    "#!pip install git+https://github.com/doctorado-ml/stree"
+    "#!pip install git+https://github.com/doctorado-ml/stree\n",
+    "!pip install pandas"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {
     "colab": {},
     "colab_type": "code",
@@ -38,6 +39,10 @@
    },
    "outputs": [],
    "source": [
+    "import random\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
     "from sklearn.ensemble import AdaBoostClassifier\n",
     "from sklearn.svm import LinearSVC\n",
     "from sklearn.model_selection import GridSearchCV, train_test_split\n",
@@ -46,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {
     "colab": {},
     "colab_type": "code",
@@ -54,7 +59,6 @@
    },
    "outputs": [],
    "source": [
-    "import os\n",
     "if not os.path.isfile('data/creditcard.csv'):\n",
     "    !wget --no-check-certificate --content-disposition http://nube.jccm.es/index.php/s/Zs7SYtZQJ3RQ2H2/download\n",
     "    !tar xzf creditcard.tgz"
@@ -62,7 +66,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {
     "colab": {},
     "colab_type": "code",
@@ -70,26 +74,11 @@
     "outputId": "afc822fb-f16a-4302-8a67-2b9e2880159b",
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Fraud: 0.173% 492\n",
-      "Valid: 99.827% 284315\n",
-      "X.shape (1492, 28)  y.shape (1492,)\n",
-      "Fraud: 33.177% 495\n",
-      "Valid: 66.823% 997\n"
-     ]
-    }
-   ],
    "source": [
     "random_state=1\n",
     "\n",
     "def load_creditcard(n_examples=0):\n",
-    "    import pandas as pd\n",
-    "    import numpy as np\n",
-    "    import random\n",
     "    df = pd.read_csv('data/creditcard.csv')\n",
     "    print(\"Fraud: {0:.3f}% {1}\".format(df.Class[df.Class == 1].count()*100/df.shape[0], df.Class[df.Class == 1].count()))\n",
     "    print(\"Valid: {0:.3f}% {1}\".format(df.Class[df.Class == 0].count()*100/df.shape[0], df.Class[df.Class == 0].count()))\n",
@@ -132,7 +121,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {
     "colab": {},
     "colab_type": "code",
@@ -176,39 +165,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "data": {
-      "text/plain": [
-       "{'C': 1.0,\n",
-       " 'criterion': 'entropy',\n",
-       " 'degree': 3,\n",
-       " 'gamma': 'scale',\n",
-       " 'kernel': 'linear',\n",
-       " 'max_depth': None,\n",
-       " 'max_features': None,\n",
-       " 'max_iter': 100000.0,\n",
-       " 'min_samples_split': 0,\n",
-       " 'random_state': None,\n",
-       " 'split_criteria': 'impurity',\n",
-       " 'splitter': 'random',\n",
-       " 'tol': 0.0001}"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
     "Stree().get_params()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {
     "colab": {},
     "colab_type": "code",
@@ -216,69 +182,7 @@
     "outputId": "7703413a-d563-4289-a13b-532f38f82762",
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Fitting 5 folds for each of 1008 candidates, totalling 5040 fits\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 16 concurrent workers.\n",
-      "[Parallel(n_jobs=-1)]: Done  40 tasks      | elapsed:    1.6s\n",
-      "[Parallel(n_jobs=-1)]: Done 130 tasks      | elapsed:    3.1s\n",
-      "[Parallel(n_jobs=-1)]: Done 256 tasks      | elapsed:    5.5s\n",
-      "[Parallel(n_jobs=-1)]: Done 418 tasks      | elapsed:    9.3s\n",
-      "[Parallel(n_jobs=-1)]: Done 616 tasks      | elapsed:   18.6s\n",
-      "[Parallel(n_jobs=-1)]: Done 850 tasks      | elapsed:   28.2s\n",
-      "[Parallel(n_jobs=-1)]: Done 1120 tasks      | elapsed:   35.4s\n",
-      "[Parallel(n_jobs=-1)]: Done 1426 tasks      | elapsed:   43.5s\n",
-      "[Parallel(n_jobs=-1)]: Done 1768 tasks      | elapsed:   51.3s\n",
-      "[Parallel(n_jobs=-1)]: Done 2146 tasks      | elapsed:  1.0min\n",
-      "[Parallel(n_jobs=-1)]: Done 2560 tasks      | elapsed:  1.2min\n",
-      "[Parallel(n_jobs=-1)]: Done 3010 tasks      | elapsed:  1.4min\n",
-      "[Parallel(n_jobs=-1)]: Done 3496 tasks      | elapsed:  1.7min\n",
-      "[Parallel(n_jobs=-1)]: Done 4018 tasks      | elapsed:  2.1min\n",
-      "[Parallel(n_jobs=-1)]: Done 4576 tasks      | elapsed:  2.6min\n",
-      "[Parallel(n_jobs=-1)]: Done 5040 out of 5040 | elapsed:  2.9min finished\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "GridSearchCV(estimator=AdaBoostClassifier(algorithm='SAMME', random_state=1),\n",
-       "             n_jobs=-1,\n",
-       "             param_grid=[{'base_estimator': [Stree(C=55, max_depth=7,\n",
-       "                                                   random_state=1,\n",
-       "                                                   split_criteria='max_samples',\n",
-       "                                                   tol=0.1)],\n",
-       "                          'base_estimator__C': [1, 7, 55],\n",
-       "                          'base_estimator__kernel': ['linear'],\n",
-       "                          'base_estimator__max_depth': [3, 5, 7],\n",
-       "                          'base_estimator__split_criteria': ['max_samples',\n",
-       "                                                             'impuri...\n",
-       "                         {'base_estimator': [Stree(random_state=1)],\n",
-       "                          'base_estimator__C': [1, 7, 55],\n",
-       "                          'base_estimator__gamma': [0.1, 1, 10],\n",
-       "                          'base_estimator__kernel': ['rbf'],\n",
-       "                          'base_estimator__max_depth': [3, 5, 7],\n",
-       "                          'base_estimator__split_criteria': ['max_samples',\n",
-       "                                                             'impurity'],\n",
-       "                          'base_estimator__tol': [0.1, 0.01],\n",
-       "                          'learning_rate': [0.5, 1],\n",
-       "                          'n_estimators': [10, 25]}],\n",
-       "             return_train_score=True, verbose=5)"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
     "clf = AdaBoostClassifier(random_state=random_state, algorithm=\"SAMME\")\n",
     "grid = GridSearchCV(clf, parameters, verbose=5, n_jobs=-1, return_train_score=True)\n",
@@ -287,7 +191,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "metadata": {
     "colab": {},
     "colab_type": "code",
@@ -295,20 +199,7 @@
     "outputId": "285163c8-fa33-4915-8ae7-61c4f7844344",
     "tags": []
    },
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Best estimator:  AdaBoostClassifier(algorithm='SAMME',\n",
-      "                   base_estimator=Stree(C=55, max_depth=7, random_state=1,\n",
-      "                                        split_criteria='max_samples', tol=0.1),\n",
-      "                   learning_rate=0.5, n_estimators=25, random_state=1)\n",
-      "Best hyperparameters:  {'base_estimator': Stree(C=55, max_depth=7, random_state=1, split_criteria='max_samples', tol=0.1), 'base_estimator__C': 55, 'base_estimator__kernel': 'linear', 'base_estimator__max_depth': 7, 'base_estimator__split_criteria': 'max_samples', 'base_estimator__tol': 0.1, 'learning_rate': 0.5, 'n_estimators': 25}\n",
-      "Best accuracy:  0.9511777695988222\n"
-     ]
-    }
-   ],
    "source": [
     "print(\"Best estimator: \", grid.best_estimator_)\n",
     "print(\"Best hyperparameters: \", grid.best_params_)\n",
@@ -354,7 +245,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.8.2-final"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -1,4 +1,2 @@
-numpy
+scikit-learn>0.24
-scikit-learn
+mufs
-pandas
-ipympl

setup.py

@@ -1,7 +1,5 @@
 import setuptools
-
+import os
-__version__ = "1.0rc1"
-__author__ = "Ricardo Montañana Gómez"
 
 
 def readme():
@@ -9,28 +7,46 @@ def readme():
         return f.read()
 
 
+def get_data(field):
+    item = ""
+    file_name = "_version.py" if field == "version" else "__init__.py"
+    with open(os.path.join("stree", file_name)) as f:
+        for line in f.readlines():
+            if line.startswith(f"__{field}__"):
+                delim = '"' if '"' in line else "'"
+                item = line.split(delim)[1]
+                break
+        else:
+            raise RuntimeError(f"Unable to find {field} string.")
+    return item
+
+
 setuptools.setup(
     name="STree",
-    version=__version__,
+    version=get_data("version"),
-    license="MIT License",
+    license=get_data("license"),
     description="Oblique decision tree with svm nodes",
     long_description=readme(),
     long_description_content_type="text/markdown",
     packages=setuptools.find_packages(),
-    url="https://github.com/doctorado-ml/stree",
+    url="https://github.com/Doctorado-ML/STree#stree",
-    author=__author__,
+    project_urls={
-    author_email="ricardo.montanana@alu.uclm.es",
+        "Code": "https://github.com/Doctorado-ML/STree",
+        "Documentation": "https://stree.readthedocs.io/en/latest/index.html",
+    },
+    author=get_data("author"),
+    author_email=get_data("author_email"),
     keywords="scikit-learn oblique-classifier oblique-decision-tree decision-\
     tree svm svc",
     classifiers=[
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
-        "License :: OSI Approved :: MIT License",
+        "License :: OSI Approved :: " + get_data("license"),
         "Programming Language :: Python :: 3.8",
         "Natural Language :: English",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",
         "Intended Audience :: Science/Research",
     ],
-    install_requires=["scikit-learn", "numpy", "ipympl"],
+    install_requires=["scikit-learn", "mufs"],
     test_suite="stree.tests",
     zip_safe=False,
 )

stree/.readthedocs.yaml

@@ -0,0 +1,10 @@
+version: 2
+
+sphinx:
+  configuration: docs/source/conf.py
+
+python:
+  version: 3.8
+  install:
+    - requirements: requirements.txt
+    - requirements: docs/requirements.txt

stree/Splitter.py

@@ -0,0 +1,809 @@
+"""
+Oblique decision tree classifier based on SVM nodes
+Splitter class
+"""
+
+import os
+import warnings
+import random
+from math import log, factorial
+import numpy as np
+from sklearn.feature_selection import SelectKBest, mutual_info_classif
+from sklearn.preprocessing import StandardScaler
+from sklearn.svm import SVC
+from sklearn.exceptions import ConvergenceWarning
+from mufs import MUFS
+
+
+class Snode:
+    """
+    Nodes of the tree that keeps the svm classifier and if testing the
+    dataset assigned to it
+
+    Parameters
+    ----------
+    clf : SVC
+        Classifier used
+    X : np.ndarray
+        input dataset in train time (only in testing)
+    y : np.ndarray
+        input labes in train time
+    features : np.array
+        features used to compute hyperplane
+    impurity : float
+        impurity of the node
+    title : str
+        label describing the route to the node
+    weight : np.ndarray, optional
+        weights applied to input dataset in train time, by default None
+    scaler : StandardScaler, optional
+        scaler used if any, by default None
+    """
+
+    def __init__(
+        self,
+        clf: SVC,
+        X: np.ndarray,
+        y: np.ndarray,
+        features: np.array,
+        impurity: float,
+        title: str,
+        weight: np.ndarray = None,
+        scaler: StandardScaler = None,
+    ):
+        self._clf = clf
+        self._title = title
+        self._belief = 0.0
+        # Only store dataset in Testing
+        self._X = X if os.environ.get("TESTING", "NS") != "NS" else None
+        self._y = y
+        self._down = None
+        self._up = None
+        self._class = None
+        self._feature = None
+        self._sample_weight = (
+            weight if os.environ.get("TESTING", "NS") != "NS" else None
+        )
+        self._features = features
+        self._impurity = impurity
+        self._partition_column: int = -1
+        self._scaler = scaler
+        self._proba = None
+
+    @classmethod
+    def copy(cls, node: "Snode") -> "Snode":
+        return cls(
+            node._clf,
+            node._X,
+            node._y,
+            node._features,
+            node._impurity,
+            node._title,
+            node._sample_weight,
+            node._scaler,
+        )
+
+    def set_partition_column(self, col: int):
+        self._partition_column = col
+
+    def get_partition_column(self) -> int:
+        return self._partition_column
+
+    def set_down(self, son):
+        self._down = son
+
+    def set_title(self, title):
+        self._title = title
+
+    def set_classifier(self, clf):
+        self._clf = clf
+
+    def set_features(self, features):
+        self._features = features
+
+    def set_impurity(self, impurity):
+        self._impurity = impurity
+
+    def get_title(self) -> str:
+        return self._title
+
+    def get_classifier(self) -> SVC:
+        return self._clf
+
+    def get_impurity(self) -> float:
+        return self._impurity
+
+    def get_features(self) -> np.array:
+        return self._features
+
+    def set_up(self, son):
+        self._up = son
+
+    def is_leaf(self) -> bool:
+        return self._up is None and self._down is None
+
+    def get_down(self) -> "Snode":
+        return self._down
+
+    def get_up(self) -> "Snode":
+        return self._up
+
+    def make_predictor(self, num_classes: int) -> None:
+        """Compute the class of the predictor and its belief based on the
+        subdataset of the node only if it is a leaf
+        """
+        if not self.is_leaf():
+            return
+        classes, card = np.unique(self._y, return_counts=True)
+        self._proba = np.zeros((num_classes,), dtype=np.int64)
+        for c, n in zip(classes, card):
+            self._proba[c] = n
+        try:
+            max_card = max(card)
+            self._class = classes[card == max_card][0]
+            self._belief = max_card / np.sum(card)
+        except ValueError:
+            self._class = None
+
+    def graph(self):
+        """
+        Return a string representing the node in graphviz format
+        """
+        output = ""
+        count_values = np.unique(self._y, return_counts=True)
+        if self.is_leaf():
+            output += (
+                f'N{id(self)} [shape=box style=filled label="'
+                f"class={self._class} impurity={self._impurity:.3f} "
+                f'counts={self._proba}"];\n'
+            )
+        else:
+            output += (
+                f'N{id(self)} [label="#features={len(self._features)} '
+                f"classes={count_values[0]} samples={count_values[1]} "
+                f'({sum(count_values[1])})" fontcolor=black];\n'
+            )
+            output += f"N{id(self)} -> N{id(self.get_up())} [color=black];\n"
+            output += f"N{id(self)} -> N{id(self.get_down())} [color=black];\n"
+        return output
+
+    def __str__(self) -> str:
+        count_values = np.unique(self._y, return_counts=True)
+        if self.is_leaf():
+            return (
+                f"{self._title} - Leaf class={self._class} belief="
+                f"{self._belief: .6f} impurity={self._impurity:.4f} "
+                f"counts={count_values}"
+            )
+        return (
+            f"{self._title} feaures={self._features} impurity="
+            f"{self._impurity:.4f} "
+            f"counts={count_values}"
+        )
+
+
+class Siterator:
+    """Stree preorder iterator"""
+
+    def __init__(self, tree: Snode):
+        self._stack = []
+        self._push(tree)
+
+    def __iter__(self):
+        # To complete the iterator interface
+        return self
+
+    def _push(self, node: Snode):
+        if node is not None:
+            self._stack.append(node)
+
+    def __next__(self) -> Snode:
+        if len(self._stack) == 0:
+            raise StopIteration()
+        node = self._stack.pop()
+        self._push(node.get_up())
+        self._push(node.get_down())
+        return node
+
+
+class Splitter:
+    """
+    Splits a dataset in two based on different criteria
+
+    Parameters
+    ----------
+    clf : SVC, optional
+        classifier, by default None
+    criterion : str, optional
+        The function to measure the quality of a split (only used if
+        max_features != num_features). Supported criteria are “gini” for the
+        Gini impurity and “entropy” for the information gain., by default
+        "entropy", by default None
+    feature_select : str, optional
+        The strategy used to choose the feature set at each node (only used if
+        max_features < num_features). Supported strategies are: “best”: sklearn
+        SelectKBest algorithm is used in every node to choose the max_features
+        best features. “random”: The algorithm generates 5 candidates and
+        choose the best (max. info. gain) of them. “trandom”: The algorithm
+        generates only one random combination. "mutual": Chooses the best
+        features w.r.t. their mutual info with the label. "cfs": Apply
+        Correlation-based Feature Selection. "fcbf": Apply Fast Correlation-
+        Based, by default None
+    criteria : str, optional
+        ecides (just in case of a multi class classification) which column
+        (class) use to split the dataset in a node. max_samples is
+        incompatible with 'ovo' multiclass_strategy, by default None
+    min_samples_split : int, optional
+        The minimum number of samples required to split an internal node. 0
+        (default) for any, by default None
+    random_state : optional
+        Controls the pseudo random number generation for shuffling the data for
+        probability estimates. Ignored when probability is False.Pass an int
+        for reproducible output across multiple function calls, by
+        default None
+    normalize : bool, optional
+        If standardization of features should be applied on each node with the
+        samples that reach it , by default False
+
+    Raises
+    ------
+    ValueError
+        clf has to be a sklearn estimator
+    ValueError
+        criterion must be gini or entropy
+    ValueError
+        criteria has to be max_samples or impurity
+    ValueError
+        splitter must be in {random, best, mutual, cfs, fcbf}
+    """
+
+    def __init__(
+        self,
+        clf: SVC = None,
+        criterion: str = None,
+        feature_select: str = None,
+        criteria: str = None,
+        min_samples_split: int = None,
+        random_state=None,
+        normalize=False,
+    ):
+
+        self._clf = clf
+        self._random_state = random_state
+        if random_state is not None:
+            random.seed(random_state)
+        self._criterion = criterion
+        self._min_samples_split = min_samples_split
+        self._criteria = criteria
+        self._feature_select = feature_select
+        self._normalize = normalize
+
+        if clf is None:
+            raise ValueError(f"clf has to be a sklearn estimator, got({clf})")
+
+        if criterion not in ["gini", "entropy"]:
+            raise ValueError(
+                f"criterion must be gini or entropy got({criterion})"
+            )
+
+        if criteria not in [
+            "max_samples",
+            "impurity",
+        ]:
+            raise ValueError(
+                f"criteria has to be max_samples or impurity; got ({criteria})"
+            )
+
+        if feature_select not in [
+            "random",
+            "trandom",
+            "best",
+            "mutual",
+            "cfs",
+            "fcbf",
+            "iwss",
+        ]:
+            raise ValueError(
+                "splitter must be in {random, trandom, best, mutual, cfs, "
+                "fcbf, iwss} "
+                f"got ({feature_select})"
+            )
+        self.criterion_function = getattr(self, f"_{self._criterion}")
+        self.decision_criteria = getattr(self, f"_{self._criteria}")
+        self.fs_function = getattr(self, f"_fs_{self._feature_select}")
+
+    def _fs_random(
+        self, dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Return the best of five random feature set combinations
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        # Random feature reduction
+        n_features = dataset.shape[1]
+        features_sets = self._generate_spaces(n_features, max_features)
+        return self._select_best_set(dataset, labels, features_sets)
+
+    @staticmethod
+    def _fs_trandom(
+        dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Return the a random feature set combination
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        # Random feature reduction
+        n_features = dataset.shape[1]
+        return tuple(sorted(random.sample(range(n_features), max_features)))
+
+    @staticmethod
+    def _fs_best(
+        dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Return the variabes with higher f-score
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        return (
+            SelectKBest(k=max_features)
+            .fit(dataset, labels)
+            .get_support(indices=True)
+        )
+
+    def _fs_mutual(
+        self, dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Return the best features with mutual information with labels
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        # return best features with mutual info with the label
+        feature_list = mutual_info_classif(
+            dataset, labels, random_state=self._random_state
+        )
+        return tuple(
+            sorted(
+                range(len(feature_list)), key=lambda sub: feature_list[sub]
+            )[-max_features:]
+        )
+
+    @staticmethod
+    def _fs_cfs(
+        dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Correlattion-based feature selection with max_features limit
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        mufs = MUFS(max_features=max_features, discrete=False)
+        return mufs.cfs(dataset, labels).get_results()
+
+    @staticmethod
+    def _fs_fcbf(
+        dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Fast Correlation-based Filter algorithm with max_features limit
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        mufs = MUFS(max_features=max_features, discrete=False)
+        return mufs.fcbf(dataset, labels, 5e-4).get_results()
+
+    @staticmethod
+    def _fs_iwss(
+        dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Correlattion-based feature selection based on iwss with max_features
+        limit
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (< number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        mufs = MUFS(max_features=max_features, discrete=False)
+        return mufs.iwss(dataset, labels, 0.25).get_results()
+
+    def partition_impurity(self, y: np.array) -> np.array:
+        return self.criterion_function(y)
+
+    @staticmethod
+    def _gini(y: np.array) -> float:
+        _, count = np.unique(y, return_counts=True)
+        return 1 - np.sum(np.square(count / np.sum(count)))
+
+    @staticmethod
+    def _entropy(y: np.array) -> float:
+        """Compute entropy of a labels set
+
+        Parameters
+        ----------
+        y : np.array
+            set of labels
+
+        Returns
+        -------
+        float
+            entropy
+        """
+        n_labels = len(y)
+        if n_labels <= 1:
+            return 0
+        counts = np.bincount(y)
+        proportions = counts / n_labels
+        n_classes = np.count_nonzero(proportions)
+        if n_classes <= 1:
+            return 0
+        entropy = 0.0
+        # Compute standard entropy.
+        for prop in proportions:
+            if prop != 0.0:
+                entropy -= prop * log(prop, n_classes)
+        return entropy
+
+    def information_gain(
+        self, labels: np.array, labels_up: np.array, labels_dn: np.array
+    ) -> float:
+        """Compute information gain of a split candidate
+
+        Parameters
+        ----------
+        labels : np.array
+            labels of the dataset
+        labels_up : np.array
+            labels of one side
+        labels_dn : np.array
+            labels on the other side
+
+        Returns
+        -------
+        float
+            information gain
+        """
+        imp_prev = self.criterion_function(labels)
+        card_up = card_dn = imp_up = imp_dn = 0
+        if labels_up is not None:
+            card_up = labels_up.shape[0]
+            imp_up = self.criterion_function(labels_up)
+        if labels_dn is not None:
+            card_dn = labels_dn.shape[0] if labels_dn is not None else 0
+            imp_dn = self.criterion_function(labels_dn)
+        samples = card_up + card_dn
+        if samples == 0:
+            return 0.0
+        else:
+            result = (
+                imp_prev
+                - (card_up / samples) * imp_up
+                - (card_dn / samples) * imp_dn
+            )
+            return result
+
+    def _select_best_set(
+        self, dataset: np.array, labels: np.array, features_sets: list
+    ) -> list:
+        """Return the best set of features among feature_sets, the criterion is
+        the information gain
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples (# samples, # features)
+        labels : np.array
+            array of labels
+        features_sets : list
+            list of features sets to check
+
+        Returns
+        -------
+        list
+            best feature set
+        """
+        max_gain = 0
+        selected = None
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        for feature_set in features_sets:
+            self._clf.fit(dataset[:, feature_set], labels)
+            node = Snode(
+                self._clf, dataset, labels, feature_set, 0.0, "subset"
+            )
+            self.partition(dataset, node, train=True)
+            y1, y2 = self.part(labels)
+            gain = self.information_gain(labels, y1, y2)
+            if gain > max_gain:
+                max_gain = gain
+                selected = feature_set
+        return selected if selected is not None else feature_set
+
+    @staticmethod
+    def _generate_spaces(features: int, max_features: int) -> list:
+        """Generate at most 5 feature random combinations
+
+        Parameters
+        ----------
+        features : int
+            number of features in each combination
+        max_features : int
+            number of features in dataset
+
+        Returns
+        -------
+        list
+            list with up to 5 combination of features randomly selected
+        """
+        comb = set()
+        # Generate at most 5 combinations
+        number = factorial(features) / (
+            factorial(max_features) * factorial(features - max_features)
+        )
+        set_length = min(5, number)
+        while len(comb) < set_length:
+            comb.add(
+                tuple(sorted(random.sample(range(features), max_features)))
+            )
+        return list(comb)
+
+    def _get_subspaces_set(
+        self, dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Compute the indices of the features selected by splitter depending
+        on the self._feature_select hyper parameter
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features of the subspace
+            (<= number of features in dataset)
+
+        Returns
+        -------
+        tuple
+            indices of the features selected
+        """
+        # No feature reduction
+        n_features = dataset.shape[1]
+        if n_features == max_features:
+            return tuple(range(n_features))
+        # select features as selected in constructor
+        return self.fs_function(dataset, labels, max_features)
+
+    def get_subspace(
+        self, dataset: np.array, labels: np.array, max_features: int
+    ) -> tuple:
+        """Re3turn a subspace of the selected dataset of max_features length.
+        Depending on hyperparameter
+
+        Parameters
+        ----------
+        dataset : np.array
+            array of samples (# samples, # features)
+        labels : np.array
+            labels of the dataset
+        max_features : int
+            number of features to form the subspace
+
+        Returns
+        -------
+        tuple
+            tuple with the dataset with only the features selected  and the
+            indices of the features selected
+        """
+        indices = self._get_subspaces_set(dataset, labels, max_features)
+        return dataset[:, indices], indices
+
+    def _impurity(self, data: np.array, y: np.array) -> np.array:
+        """return column of dataset to be taken into account to split dataset
+
+        Parameters
+        ----------
+        data : np.array
+            distances to hyper plane of every class
+        y : np.array
+            vector of labels (classes)
+
+        Returns
+        -------
+        np.array
+            column of dataset to be taken into account to split dataset
+        """
+        max_gain = 0
+        selected = -1
+        for col in range(data.shape[1]):
+            tup = y[data[:, col] > 0]
+            tdn = y[data[:, col] <= 0]
+            info_gain = self.information_gain(y, tup, tdn)
+            if info_gain > max_gain:
+                selected = col
+                max_gain = info_gain
+        return selected
+
+    @staticmethod
+    def _max_samples(data: np.array, y: np.array) -> np.array:
+        """return column of dataset to be taken into account to split dataset
+
+        Parameters
+        ----------
+        data : np.array
+            distances to hyper plane of every class
+        y : np.array
+            column of dataset to be taken into account to split dataset
+
+        Returns
+        -------
+        np.array
+            column of dataset to be taken into account to split dataset
+        """
+        # select the class with max number of samples
+        _, samples = np.unique(y, return_counts=True)
+        return np.argmax(samples)
+
+    def partition(self, samples: np.array, node: Snode, train: bool):
+        """Set the criteria to split arrays. Compute the indices of the samples
+        that should go to one side of the tree (up)
+
+        Parameters
+        ----------
+        samples : np.array
+            array of samples (# samples, # features)
+        node : Snode
+            Node of the tree where partition is going to be made
+        train : bool
+            Train time - True / Test time - False
+        """
+        # data contains the distances of every sample to every class hyperplane
+        # array of (m, nc) nc = # classes
+        data = self._distances(node, samples)
+        if data.shape[0] < self._min_samples_split:
+            # there aren't enough samples to split
+            self._up = np.ones((data.shape[0]), dtype=bool)
+            return
+        if data.ndim > 1:
+            # split criteria for multiclass
+            # Convert data to a (m, 1) array selecting values for samples
+            if train:
+                # in train time we have to compute the column to take into
+                # account to split the dataset
+                col = self.decision_criteria(data, node._y)
+                node.set_partition_column(col)
+            else:
+                # in predcit time just use the column computed in train time
+                # is taking the classifier of class <col>
+                col = node.get_partition_column()
+            if col == -1:
+                # No partition is producing information gain
+                data = np.ones(data.shape)
+            data = data[:, col]
+        self._up = data > 0
+
+    def part(self, origin: np.array) -> list:
+        """Split an array in two based on indices (self._up) and its complement
+        partition has to be called first to establish up indices
+
+        Parameters
+        ----------
+        origin : np.array
+            dataset to split
+
+        Returns
+        -------
+        list
+            list with two splits of the array
+        """
+        down = ~self._up
+        return [
+            origin[self._up] if any(self._up) else None,
+            origin[down] if any(down) else None,
+        ]
+
+    def _distances(self, node: Snode, data: np.ndarray) -> np.array:
+        """Compute distances of the samples to the hyperplane of the node
+
+        Parameters
+        ----------
+        node : Snode
+            node containing the svm classifier
+        data : np.ndarray
+            samples to compute distance to hyperplane
+
+        Returns
+        -------
+        np.array
+            array of shape (m, nc) with the distances of every sample to
+            the hyperplane of every class. nc = # of classes
+        """
+        X_transformed = data[:, node._features]
+        if self._normalize:
+            X_transformed = node._scaler.transform(X_transformed)
+        return node._clf.decision_function(X_transformed)

stree/Strees.py

@@ -1,493 +1,138 @@
 """
-__author__ = "Ricardo Montañana Gómez"
+Oblique decision tree classifier based on SVM nodes
-__copyright__ = "Copyright 2020, Ricardo Montañana Gómez"
-__license__ = "MIT"
-__version__ = "0.9"
-Build an oblique tree classifier based on SVM nodes
 """
 
-import os
 import numbers
 import random
-import warnings
-from math import log, factorial
 from typing import Optional
 import numpy as np
 from sklearn.base import BaseEstimator, ClassifierMixin
 from sklearn.svm import SVC, LinearSVC
-from sklearn.utils import check_consistent_length
+from sklearn.preprocessing import StandardScaler
 from sklearn.utils.multiclass import check_classification_targets
-from sklearn.exceptions import ConvergenceWarning
 from sklearn.utils.validation import (
     check_X_y,
     check_array,
     check_is_fitted,
     _check_sample_weight,
 )
-from sklearn.metrics._classification import _weighted_sum, _check_targets
+from .Splitter import Splitter, Snode, Siterator
-
+from ._version import __version__
-
-class Snode:
-    """Nodes of the tree that keeps the svm classifier and if testing the
-    dataset assigned to it
-    """
-
-    def __init__(
-        self,
-        clf: SVC,
-        X: np.ndarray,
-        y: np.ndarray,
-        features: np.array,
-        impurity: float,
-        title: str,
-        weight: np.ndarray = None,
-    ):
-        self._clf = clf
-        self._title = title
-        self._belief = 0.0
-        # Only store dataset in Testing
-        self._X = X if os.environ.get("TESTING", "NS") != "NS" else None
-        self._y = y
-        self._down = None
-        self._up = None
-        self._class = None
-        self._feature = None
-        self._sample_weight = (
-            weight if os.environ.get("TESTING", "NS") != "NS" else None
-        )
-        self._features = features
-        self._impurity = impurity
-        self._partition_column: int = -1
-
-    @classmethod
-    def copy(cls, node: "Snode") -> "Snode":
-        return cls(
-            node._clf,
-            node._X,
-            node._y,
-            node._features,
-            node._impurity,
-            node._title,
-        )
-
-    def set_partition_column(self, col: int):
-        self._partition_column = col
-
-    def get_partition_column(self) -> int:
-        return self._partition_column
-
-    def set_down(self, son):
-        self._down = son
-
-    def set_up(self, son):
-        self._up = son
-
-    def is_leaf(self) -> bool:
-        return self._up is None and self._down is None
-
-    def get_down(self) -> "Snode":
-        return self._down
-
-    def get_up(self) -> "Snode":
-        return self._up
-
-    def make_predictor(self):
-        """Compute the class of the predictor and its belief based on the
-        subdataset of the node only if it is a leaf
-        """
-        if not self.is_leaf():
-            return
-        classes, card = np.unique(self._y, return_counts=True)
-        if len(classes) > 1:
-            max_card = max(card)
-            self._class = classes[card == max_card][0]
-            self._belief = max_card / np.sum(card)
-        else:
-            self._belief = 1
-            try:
-                self._class = classes[0]
-            except IndexError:
-                self._class = None
-
-    def __str__(self) -> str:
-        count_values = np.unique(self._y, return_counts=True)
-        if self.is_leaf():
-            return (
-                f"{self._title} - Leaf class={self._class} belief="
-                f"{self._belief: .6f} impurity={self._impurity:.4f} "
-                f"counts={count_values}"
-            )
-        else:
-            return (
-                f"{self._title} feaures={self._features} impurity="
-                f"{self._impurity:.4f} "
-                f"counts={count_values}"
-            )
-
-
-class Siterator:
-    """Stree preorder iterator"""
-
-    def __init__(self, tree: Snode):
-        self._stack = []
-        self._push(tree)
-
-    def _push(self, node: Snode):
-        if node is not None:
-            self._stack.append(node)
-
-    def __next__(self) -> Snode:
-        if len(self._stack) == 0:
-            raise StopIteration()
-        node = self._stack.pop()
-        self._push(node.get_up())
-        self._push(node.get_down())
-        return node
-
-
-class Splitter:
-    def __init__(
-        self,
-        clf: SVC = None,
-        criterion: str = None,
-        splitter_type: str = None,
-        criteria: str = None,
-        min_samples_split: int = None,
-        random_state=None,
-    ):
-        self._clf = clf
-        self._random_state = random_state
-        if random_state is not None:
-            random.seed(random_state)
-        self._criterion = criterion
-        self._min_samples_split = min_samples_split
-        self._criteria = criteria
-        self._splitter_type = splitter_type
-
-        if clf is None:
-            raise ValueError(f"clf has to be a sklearn estimator, got({clf})")
-
-        if criterion not in ["gini", "entropy"]:
-            raise ValueError(
-                f"criterion must be gini or entropy got({criterion})"
-            )
-
-        if criteria not in [
-            "max_samples",
-            "impurity",
-        ]:
-            raise ValueError(
-                f"criteria has to be max_samples or impurity; got ({criteria})"
-            )
-
-        if splitter_type not in ["random", "best"]:
-            raise ValueError(
-                f"splitter must be either random or best, got({splitter_type})"
-            )
-        self.criterion_function = getattr(self, f"_{self._criterion}")
-        self.decision_criteria = getattr(self, f"_{self._criteria}")
-
-    def partition_impurity(self, y: np.array) -> np.array:
-        return self.criterion_function(y)
-
-    @staticmethod
-    def _gini(y: np.array) -> float:
-        _, count = np.unique(y, return_counts=True)
-        return 1 - np.sum(np.square(count / np.sum(count)))
-
-    @staticmethod
-    def _entropy(y: np.array) -> float:
-        """Compute entropy of a labels set
-
-        Parameters
-        ----------
-        y : np.array
-            set of labels
-
-        Returns
-        -------
-        float
-            entropy
-        """
-        n_labels = len(y)
-        if n_labels <= 1:
-            return 0
-        counts = np.bincount(y)
-        proportions = counts / n_labels
-        n_classes = np.count_nonzero(proportions)
-        if n_classes <= 1:
-            return 0
-        entropy = 0.0
-        # Compute standard entropy.
-        for prop in proportions:
-            if prop != 0.0:
-                entropy -= prop * log(prop, n_classes)
-        return entropy
-
-    def information_gain(
-        self, labels: np.array, labels_up: np.array, labels_dn: np.array
-    ) -> float:
-        """Compute information gain of a split candidate
-
-        Parameters
-        ----------
-        labels : np.array
-            labels of the dataset
-        labels_up : np.array
-            labels of one side
-        labels_dn : np.array
-            labels on the other side
-
-        Returns
-        -------
-        float
-            information gain
-        """
-        imp_prev = self.criterion_function(labels)
-        card_up = card_dn = imp_up = imp_dn = 0
-        if labels_up is not None:
-            card_up = labels_up.shape[0]
-            imp_up = self.criterion_function(labels_up)
-        if labels_dn is not None:
-            card_dn = labels_dn.shape[0] if labels_dn is not None else 0
-            imp_dn = self.criterion_function(labels_dn)
-        samples = card_up + card_dn
-        if samples == 0:
-            return 0.0
-        else:
-            result = (
-                imp_prev
-                - (card_up / samples) * imp_up
-                - (card_dn / samples) * imp_dn
-            )
-            return result
-
-    def _select_best_set(
-        self, dataset: np.array, labels: np.array, features_sets: list
-    ) -> list:
-        max_gain = 0
-        selected = None
-        warnings.filterwarnings("ignore", category=ConvergenceWarning)
-        for feature_set in features_sets:
-            self._clf.fit(dataset[:, feature_set], labels)
-            node = Snode(
-                self._clf, dataset, labels, feature_set, 0.0, "subset"
-            )
-            self.partition(dataset, node, train=True)
-            y1, y2 = self.part(labels)
-            gain = self.information_gain(labels, y1, y2)
-            if gain > max_gain:
-                max_gain = gain
-                selected = feature_set
-        return selected if selected is not None else feature_set
-
-    @staticmethod
-    def _generate_spaces(features: int, max_features: int) -> list:
-        """Generate at most 5 feature random combinations
-
-        Parameters
-        ----------
-        features : int
-            number of features in each combination
-        max_features : int
-            number of features in dataset
-
-        Returns
-        -------
-        list
-            list with up to 5 combination of features randomly selected
-        """
-        comb = set()
-        # Generate at most 5 combinations
-        if max_features == features:
-            set_length = 1
-        else:
-            number = factorial(features) / (
-                factorial(max_features) * factorial(features - max_features)
-            )
-            set_length = min(5, number)
-        while len(comb) < set_length:
-            comb.add(
-                tuple(sorted(random.sample(range(features), max_features)))
-            )
-        return list(comb)
-
-    def _get_subspaces_set(
-        self, dataset: np.array, labels: np.array, max_features: int
-    ) -> np.array:
-        """Compute the indices of the features selected by splitter depending
-        on the self._splitter_type hyper parameter
-
-        Parameters
-        ----------
-        dataset : np.array
-            array of samples
-        labels : np.array
-            labels of the dataset
-        max_features : int
-            number of features of the subspace
-            (<= number of features in dataset)
-
-        Returns
-        -------
-        np.array
-            indices of the features selected
-        """
-        features_sets = self._generate_spaces(dataset.shape[1], max_features)
-        if len(features_sets) > 1:
-            if self._splitter_type == "random":
-                index = random.randint(0, len(features_sets) - 1)
-                return features_sets[index]
-            else:
-                return self._select_best_set(dataset, labels, features_sets)
-        else:
-            return features_sets[0]
-
-    def get_subspace(
-        self, dataset: np.array, labels: np.array, max_features: int
-    ) -> tuple:
-        """Return a subspace of the selected dataset of max_features length.
-        Depending on hyperparmeter
-
-        Parameters
-        ----------
-        dataset : np.array
-            array of samples (# samples, # features)
-        labels : np.array
-            labels of the dataset
-        max_features : int
-            number of features to form the subspace
-
-        Returns
-        -------
-        tuple
-            tuple with the dataset with only the features selected  and the
-            indices of the features selected
-        """
-        indices = self._get_subspaces_set(dataset, labels, max_features)
-        return dataset[:, indices], indices
-
-    def _impurity(self, data: np.array, y: np.array) -> np.array:
-        """return column of dataset to be taken into account to split dataset
-
-        Parameters
-        ----------
-        data : np.array
-            distances to hyper plane of every class
-        y : np.array
-            vector of labels (classes)
-
-        Returns
-        -------
-        np.array
-            column of dataset to be taken into account to split dataset
-        """
-        max_gain = 0
-        selected = -1
-        for col in range(data.shape[1]):
-            tup = y[data[:, col] > 0]
-            tdn = y[data[:, col] <= 0]
-            info_gain = self.information_gain(y, tup, tdn)
-            if info_gain > max_gain:
-                selected = col
-                max_gain = info_gain
-        return selected
-
-    @staticmethod
-    def _max_samples(data: np.array, y: np.array) -> np.array:
-        """return column of dataset to be taken into account to split dataset
-
-        Parameters
-        ----------
-        data : np.array
-            distances to hyper plane of every class
-        y : np.array
-            column of dataset to be taken into account to split dataset
-
-        Returns
-        -------
-        np.array
-            column of dataset to be taken into account to split dataset
-        """
-        # select the class with max number of samples
-        _, samples = np.unique(y, return_counts=True)
-        return np.argmax(samples)
-
-    def partition(self, samples: np.array, node: Snode, train: bool):
-        """Set the criteria to split arrays. Compute the indices of the samples
-        that should go to one side of the tree (up)
-        """
-        # data contains the distances of every sample to every class hyperplane
-        # array of (m, nc) nc = # classes
-        data = self._distances(node, samples)
-        if data.shape[0] < self._min_samples_split:
-            # there aren't enough samples to split
-            self._up = np.ones((data.shape[0]), dtype=bool)
-            return
-        if data.ndim > 1:
-            # split criteria for multiclass
-            # Convert data to a (m, 1) array selecting values for samples
-            if train:
-                # in train time we have to compute the column to take into
-                # account to split the dataset
-                col = self.decision_criteria(data, node._y)
-                node.set_partition_column(col)
-            else:
-                # in predcit time just use the column computed in train time
-                # is taking the classifier of class <col>
-                col = node.get_partition_column()
-                if col == -1:
-                    # No partition is producing information gain
-                    data = np.ones(data.shape)
-            data = data[:, col]
-        self._up = data > 0
-
-    def part(self, origin: np.array) -> list:
-        """Split an array in two based on indices (self._up) and its complement
-        partition has to be called first to establish up indices
-
-        Parameters
-        ----------
-        origin : np.array
-            dataset to split
-
-        Returns
-        -------
-        list
-            list with two splits of the array
-        """
-        down = ~self._up
-        return [
-            origin[self._up] if any(self._up) else None,
-            origin[down] if any(down) else None,
-        ]
-
-    @staticmethod
-    def _distances(node: Snode, data: np.ndarray) -> np.array:
-        """Compute distances of the samples to the hyperplane of the node
-
-        Parameters
-        ----------
-        node : Snode
-            node containing the svm classifier
-        data : np.ndarray
-            samples to compute distance to hyperplane
-
-        Returns
-        -------
-        np.array
-            array of shape (m, nc) with the distances of every sample to
-            the hyperplane of every class. nc = # of classes
-        """
-        return node._clf.decision_function(data[:, node._features])
 
 
 class Stree(BaseEstimator, ClassifierMixin):
-    """Estimator that is based on binary trees of svm nodes
+    """
+    Estimator that is based on binary trees of svm nodes
     can deal with sample_weights in predict, used in boosting sklearn methods
     inheriting from BaseEstimator implements get_params and set_params methods
     inheriting from ClassifierMixin implement the attribute _estimator_type
     with "classifier" as value
+
+    Parameters
+    ----------
+    C : float, optional
+        Regularization parameter. The strength of the regularization is
+        inversely proportional to C. Must be strictly positive., by default 1.0
+    kernel : str, optional
+        Specifies the kernel type to be used in the algorithm. It must be one
+        of ‘liblinear’, ‘linear’, ‘poly’ or ‘rbf’. liblinear uses
+        [liblinear](https://www.csie.ntu.edu.tw/~cjlin/liblinear/) library and
+        the rest uses [libsvm](https://www.csie.ntu.edu.tw/~cjlin/libsvm/)
+        library through scikit-learn library, by default "linear"
+    max_iter : int, optional
+        Hard limit on iterations within solver, or -1 for no limit., by default
+        1e5
+    random_state : int, optional
+        Controls the pseudo random number generation for shuffling the data for
+        probability estimates. Ignored when probability is False.Pass an int
+        for reproducible output across multiple function calls, by
+        default None
+    max_depth : int, optional
+        Specifies the maximum depth of the tree, by default None
+    tol : float, optional
+        Tolerance for stopping, by default 1e-4
+    degree : int, optional
+        Degree of the polynomial kernel function (‘poly’). Ignored by all other
+        kernels., by default 3
+    gamma : str, optional
+        Kernel coefficient for ‘rbf’, ‘poly’ and ‘sigmoid’.if gamma='scale'
+        (default) is passed then it uses 1 / (n_features * X.var()) as value
+        of gamma,if ‘auto’, uses 1 / n_features., by default "scale"
+    split_criteria : str, optional
+        Decides (just in case of a multi class classification) which column
+        (class) use to split the dataset in a node. max_samples is
+        incompatible with 'ovo' multiclass_strategy, by default "impurity"
+    criterion : str, optional
+        The function to measure the quality of a split (only used if
+        max_features != num_features). Supported criteria are “gini” for the
+        Gini impurity and “entropy” for the information gain., by default
+        "entropy"
+    min_samples_split : int, optional
+        The minimum number of samples required to split an internal node. 0
+        (default) for any, by default 0
+    max_features : optional
+        The number of features to consider when looking for the split: If int,
+        then consider max_features features at each split. If float, then
+        max_features is a fraction and int(max_features * n_features) features
+        are considered at each split. If “auto”, then max_features=
+        sqrt(n_features). If “sqrt”, then max_features=sqrt(n_features). If
+        “log2”, then max_features=log2(n_features). If None, then max_features=
+        n_features., by default None
+    splitter : str, optional
+        The strategy used to choose the feature set at each node (only used if
+        max_features < num_features). Supported strategies are: “best”: sklearn
+        SelectKBest algorithm is used in every node to choose the max_features
+        best features. “random”: The algorithm generates 5 candidates and
+        choose the best (max. info. gain) of them. “trandom”: The algorithm
+        generates only one random combination. "mutual": Chooses the best
+        features w.r.t. their mutual info with the label. "cfs": Apply
+        Correlation-based Feature Selection. "fcbf": Apply Fast Correlation-
+        Based , by default "random"
+    multiclass_strategy : str, optional
+        Strategy to use with multiclass datasets, "ovo": one versus one. "ovr":
+        one versus rest, by default "ovo"
+    normalize : bool, optional
+        If standardization of features should be applied on each node with the
+        samples that reach it , by default False
+
+    Attributes
+    ----------
+    classes_ : ndarray of shape (n_classes,)
+        The classes labels.
+
+    n_classes_ : int
+        The number of classes
+
+    n_iter_ : int
+        Max number of iterations in classifier
+
+    depth_ : int
+        Max depht of the tree
+
+    n_features_ : int
+        The number of features when ``fit`` is performed.
+
+    n_features_in_ : int
+        Number of features seen during :term:`fit`.
+
+    max_features_ : int
+        Number of features to use in hyperplane computation
+
+    tree_ : Node
+        root of the tree
+
+    X_ : ndarray
+        points to the input dataset
+
+    y_ : ndarray
+        points to the input labels
+
+    References
+    ----------
+    R. Montañana, J. A. Gámez, J. M. Puerta, "STree: a single multi-class
+    oblique decision tree based on support vector machines.", 2021 LNAI 12882
+
+
     """
 
     def __init__(
@@ -505,7 +150,10 @@ class Stree(BaseEstimator, ClassifierMixin):
         min_samples_split: int = 0,
         max_features=None,
         splitter: str = "random",
+        multiclass_strategy: str = "ovo",
+        normalize: bool = False,
     ):
+
         self.max_iter = max_iter
         self.C = C
         self.kernel = kernel
@@ -519,6 +167,13 @@ class Stree(BaseEstimator, ClassifierMixin):
         self.max_features = max_features
         self.criterion = criterion
         self.splitter = splitter
+        self.normalize = normalize
+        self.multiclass_strategy = multiclass_strategy
+
+    @staticmethod
+    def version() -> str:
+        """Return the version of the package."""
+        return __version__
 
     def _more_tags(self) -> dict:
         """Required by sklearn to supply features of the classifier
@@ -563,7 +218,25 @@ class Stree(BaseEstimator, ClassifierMixin):
                 f"Maximum depth has to be greater than 1... got (max_depth=\
                     {self.max_depth})"
             )
-
+        if self.multiclass_strategy not in ["ovr", "ovo"]:
+            raise ValueError(
+                "mutliclass_strategy has to be either ovr or ovo"
+                f" but got {self.multiclass_strategy}"
+            )
+        if self.multiclass_strategy == "ovo":
+            if self.kernel == "liblinear":
+                raise ValueError(
+                    "The kernel liblinear is incompatible with ovo "
+                    "multiclass_strategy"
+                )
+            if self.split_criteria == "max_samples":
+                raise ValueError(
+                    "The multiclass_strategy 'ovo' is incompatible with "
+                    "split_criteria 'max_samples'"
+                )
+        kernels = ["liblinear", "linear", "rbf", "poly", "sigmoid"]
+        if self.kernel not in kernels:
+            raise ValueError(f"Kernel {self.kernel} not in {kernels}")
         check_classification_targets(y)
         X, y = check_X_y(X, y)
         sample_weight = _check_sample_weight(
@@ -578,10 +251,11 @@ class Stree(BaseEstimator, ClassifierMixin):
         self.splitter_ = Splitter(
             clf=self._build_clf(),
             criterion=self.criterion,
-            splitter_type=self.splitter,
+            feature_select=self.splitter,
             criteria=self.split_criteria,
             random_state=self.random_state,
             min_samples_split=self.min_samples_split,
+            normalize=self.normalize,
         )
         if self.random_state is not None:
             random.seed(self.random_state)
@@ -592,13 +266,12 @@ class Stree(BaseEstimator, ClassifierMixin):
         self.n_features_ = X.shape[1]
         self.n_features_in_ = X.shape[1]
         self.max_features_ = self._initialize_max_features()
-        self.tree_ = self.train(X, y, sample_weight, 1, "root")
+        self.tree_ = self._train(X, y, sample_weight, 1, "root")
-        self._build_predictor()
         self.X_ = X
         self.y_ = y
         return self
 
-    def train(
+    def _train(
         self,
         X: np.ndarray,
         y: np.ndarray,
@@ -635,57 +308,45 @@ class Stree(BaseEstimator, ClassifierMixin):
             X = X[~indices_zero, :]
             y = y[~indices_zero]
             sample_weight = sample_weight[~indices_zero]
+        self.depth_ = max(depth, self.depth_)
+        scaler = StandardScaler()
+        node = Snode(None, X, y, X.shape[1], 0.0, title, sample_weight, scaler)
         if np.unique(y).shape[0] == 1:
             # only 1 class => pure dataset
-            return Snode(
+            node.set_title(title + ", <pure>")
-                clf=None,
+            node.make_predictor(self.n_classes_)
-                X=X,
+            return node
-                y=y,
-                features=X.shape[1],
-                impurity=0.0,
-                title=title + ", <pure>",
-                weight=sample_weight,
-            )
         # Train the model
         clf = self._build_clf()
         Xs, features = self.splitter_.get_subspace(X, y, self.max_features_)
+        if self.normalize:
+            scaler.fit(Xs)
+            Xs = scaler.transform(Xs)
         clf.fit(Xs, y, sample_weight=sample_weight)
-        impurity = self.splitter_.partition_impurity(y)
+        node.set_impurity(self.splitter_.partition_impurity(y))
-        node = Snode(clf, X, y, features, impurity, title, sample_weight)
+        node.set_classifier(clf)
-        self.depth_ = max(depth, self.depth_)
+        node.set_features(features)
         self.splitter_.partition(X, node, True)
         X_U, X_D = self.splitter_.part(X)
         y_u, y_d = self.splitter_.part(y)
         sw_u, sw_d = self.splitter_.part(sample_weight)
         if X_U is None or X_D is None:
             # didn't part anything
-            return Snode(
+            node.set_title(title + ", <cgaf>")
-                clf,
+            node.make_predictor(self.n_classes_)
-                X,
+            return node
-                y,
+        node.set_up(
-                features=X.shape[1],
+            self._train(X_U, y_u, sw_u, depth + 1, title + f" - Up({depth+1})")
-                impurity=impurity,
+        )
-                title=title + ", <cgaf>",
+        node.set_down(
-                weight=sample_weight,
+            self._train(
+                X_D, y_d, sw_d, depth + 1, title + f" - Down({depth+1})"
+            )
         )
-        node.set_up(self.train(X_U, y_u, sw_u, depth + 1, title + " - Up"))
-        node.set_down(self.train(X_D, y_d, sw_d, depth + 1, title + " - Down"))
         return node
 
-    def _build_predictor(self):
-        """Process the leaves to make them predictors"""
-
-        def run_tree(node: Snode):
-            if node.is_leaf():
-                node.make_predictor()
-                return
-            run_tree(node.get_down())
-            run_tree(node.get_up())
-
-        run_tree(self.tree_)
-
     def _build_clf(self):
-        """Build the correct classifier for the node"""
+        """Build the right classifier for the node"""
         return (
             LinearSVC(
                 max_iter=self.max_iter,
@@ -693,7 +354,7 @@ class Stree(BaseEstimator, ClassifierMixin):
                 C=self.C,
                 tol=self.tol,
             )
-            if self.kernel == "linear"
+            if self.kernel == "liblinear"
             else SVC(
                 kernel=self.kernel,
                 max_iter=self.max_iter,
@@ -701,31 +362,71 @@ class Stree(BaseEstimator, ClassifierMixin):
                 C=self.C,
                 gamma=self.gamma,
                 degree=self.degree,
+                random_state=self.random_state,
+                decision_function_shape=self.multiclass_strategy,
             )
         )
 
-    @staticmethod
+    def __predict_class(self, X: np.array) -> np.array:
-    def _reorder_results(y: np.array, indices: np.array) -> np.array:
+        def compute_prediction(xp, indices, node):
-        """Reorder an array based on the array of indices passed
+            if xp is None:
+                return
+            if node.is_leaf():
+                # set a class for indices
+                result[indices] = node._proba
+                return
+            self.splitter_.partition(xp, node, train=False)
+            x_u, x_d = self.splitter_.part(xp)
+            i_u, i_d = self.splitter_.part(indices)
+            compute_prediction(x_u, i_u, node.get_up())
+            compute_prediction(x_d, i_d, node.get_down())
+
+        # setup prediction & make it happen
+        result = np.zeros((X.shape[0], self.n_classes_))
+        indices = np.arange(X.shape[0])
+        compute_prediction(X, indices, self.tree_)
+        return result
+
+    def check_predict(self, X) -> np.array:
+        check_is_fitted(self, ["tree_"])
+        # Input validation
+        X = check_array(X)
+        if X.shape[1] != self.n_features_:
+            raise ValueError(
+                f"Expected {self.n_features_} features but got "
+                f"({X.shape[1]})"
+            )
+        return X
+
+    def predict_proba(self, X: np.array) -> np.array:
+        """Predict class probabilities of the input samples X.
+
+        The predicted class probability is the fraction of samples of the same
+        class in a leaf.
 
         Parameters
         ----------
-        y : np.array
+        X : dataset of samples.
-            data untidy
-        indices : np.array
-            indices used to set order
 
         Returns
         -------
-        np.array
+        proba : array of shape (n_samples, n_classes)
-            array y ordered
+            The class probabilities of the input samples.
+
+        Raises
+        ------
+        ValueError
+            if dataset with inconsistent number of features
+        NotFittedError
+            if model is not fitted
         """
-        # return array of same type given in y
+
-        y_ordered = y.copy()
+        X = self.check_predict(X)
-        indices = indices.astype(int)
+        # return # of samples of each class in leaf node
-        for i, index in enumerate(indices):
+        values = self.__predict_class(X)
-            y_ordered[index] = y[i]
+        normalizer = values.sum(axis=1)[:, np.newaxis]
-        return y_ordered
+        normalizer[normalizer == 0.0] = 1.0
+        return values / normalizer
 
     def predict(self, X: np.array) -> np.array:
         """Predict labels for each sample in dataset passed
@@ -747,70 +448,24 @@ class Stree(BaseEstimator, ClassifierMixin):
         NotFittedError
             if model is not fitted
         """
+        X = self.check_predict(X)
+        return self.classes_[np.argmax(self.__predict_class(X), axis=1)]
 
-        def predict_class(
+    def nodes_leaves(self) -> tuple:
-            xp: np.array, indices: np.array, node: Snode
+        """Compute the number of nodes and leaves in the built tree
-        ) -> np.array:
-            if xp is None:
-                return [], []
-            if node.is_leaf():
-                # set a class for every sample in dataset
-                prediction = np.full((xp.shape[0], 1), node._class)
-                return prediction, indices
-            self.splitter_.partition(xp, node, train=False)
-            x_u, x_d = self.splitter_.part(xp)
-            i_u, i_d = self.splitter_.part(indices)
-            prx_u, prin_u = predict_class(x_u, i_u, node.get_up())
-            prx_d, prin_d = predict_class(x_d, i_d, node.get_down())
-            return np.append(prx_u, prx_d), np.append(prin_u, prin_d)
-
-        # sklearn check
-        check_is_fitted(self, ["tree_"])
-        # Input validation
-        X = check_array(X)
-        if X.shape[1] != self.n_features_:
-            raise ValueError(
-                f"Expected {self.n_features_} features but got "
-                f"({X.shape[1]})"
-            )
-        # setup prediction & make it happen
-        indices = np.arange(X.shape[0])
-        result = (
-            self._reorder_results(*predict_class(X, indices, self.tree_))
-            .astype(int)
-            .ravel()
-        )
-        return self.classes_[result]
-
-    def score(
-        self, X: np.array, y: np.array, sample_weight: np.array = None
-    ) -> float:
-        """Compute accuracy of the prediction
-
-        Parameters
-        ----------
-        X : np.array
-            dataset of samples to make predictions
-        y : np.array
-            samples labels
-        sample_weight : np.array, optional
-            weights of the samples. Rescale C per sample, by default None
 
         Returns
         -------
-        float
+        [tuple]
-            accuracy of the prediction
+            tuple with the number of nodes and the number of leaves
         """
-        # sklearn check
+        nodes = 0
-        check_is_fitted(self)
+        leaves = 0
-        check_classification_targets(y)
+        for node in self:
-        X, y = check_X_y(X, y)
+            nodes += 1
-        y_pred = self.predict(X).reshape(y.shape)
+            if node.is_leaf():
-        # Compute accuracy for each possible representation
+                leaves += 1
-        _, y_true, y_pred = _check_targets(y, y_pred)
+        return nodes, leaves
-        check_consistent_length(y_true, y_pred, sample_weight)
-        score = y_true == y_pred
-        return _weighted_sum(score, sample_weight, normalize=True)
 
     def __iter__(self) -> Siterator:
         """Create an iterator to be able to visit the nodes of the tree in
@@ -827,6 +482,23 @@ class Stree(BaseEstimator, ClassifierMixin):
             tree = None
         return Siterator(tree)
 
+    def graph(self, title="") -> str:
+        """Graphviz code representing the tree
+
+        Returns
+        -------
+        str
+            graphviz code
+        """
+        output = (
+            "digraph STree {\nlabel=<STree "
+            f"{title}>\nfontsize=30\nfontcolor=blue\nlabelloc=t\n"
+        )
+        for node in self:
+            output += node.graph()
+        output += "}\n"
+        return output
+
     def __str__(self) -> str:
         """String representation of the tree
 
@@ -857,6 +529,12 @@ class Stree(BaseEstimator, ClassifierMixin):
         elif self.max_features is None:
             max_features = self.n_features_
         elif isinstance(self.max_features, numbers.Integral):
+            if self.max_features > self.n_features_:
+                raise ValueError(
+                    "Invalid value for max_features. "
+                    "It can not be greater than number of features "
+                    f"({self.n_features_})"
+                )
             max_features = self.max_features
         else:  # float
             if self.max_features > 0.0:

stree/__init__.py

@@ -1,3 +1,8 @@
-from .Strees import Stree, Snode, Siterator, Splitter
+from .Strees import Stree, Siterator
 
-__all__ = ["Stree", "Snode", "Siterator", "Splitter"]
+__author__ = "Ricardo Montañana Gómez"
+__copyright__ = "Copyright 2020-2021, Ricardo Montañana Gómez"
+__license__ = "MIT License"
+__author_email__ = "ricardo.montanana@alu.uclm.es"
+
+__all__ = ["Stree", "Siterator"]

stree/_version.py

@@ -0,0 +1 @@
+__version__ = "1.2.4"

stree/tests/Snode_test.py

@@ -1,16 +1,19 @@
 import os
 import unittest
-
 import numpy as np
-
+from stree import Stree
-from stree import Stree, Snode
+from stree.Splitter import Snode
 from .utils import load_dataset
 
 
 class Snode_test(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         self._random_state = 1
-        self._clf = Stree(random_state=self._random_state)
+        self._clf = Stree(
+            random_state=self._random_state,
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+        )
         self._clf.fit(*load_dataset(self._random_state))
         super().__init__(*args, **kwargs)
 
@@ -64,25 +67,53 @@ class Snode_test(unittest.TestCase):
 
     def test_make_predictor_on_leaf(self):
         test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
-        test.make_predictor()
+        test.make_predictor(2)
         self.assertEqual(1, test._class)
         self.assertEqual(0.75, test._belief)
         self.assertEqual(-1, test._partition_column)
+        self.assertListEqual([1, 3], test._proba.tolist())
 
     def test_make_predictor_on_not_leaf(self):
         test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
         test.set_up(Snode(None, [1], [1], [], 0.0, "another_test"))
-        test.make_predictor()
+        test.make_predictor(2)
         self.assertIsNone(test._class)
         self.assertEqual(0, test._belief)
         self.assertEqual(-1, test._partition_column)
         self.assertEqual(-1, test.get_up()._partition_column)
+        self.assertIsNone(test._proba)
 
     def test_make_predictor_on_leaf_bogus_data(self):
         test = Snode(None, [1, 2, 3, 4], [], [], 0.0, "test")
-        test.make_predictor()
+        test.make_predictor(2)
         self.assertIsNone(test._class)
         self.assertEqual(-1, test._partition_column)
+        self.assertListEqual([0, 0], test._proba.tolist())
+
+    def test_set_title(self):
+        test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
+        self.assertEqual("test", test.get_title())
+        test.set_title("another")
+        self.assertEqual("another", test.get_title())
+
+    def test_set_classifier(self):
+        test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
+        clf = Stree()
+        self.assertIsNone(test.get_classifier())
+        test.set_classifier(clf)
+        self.assertEqual(clf, test.get_classifier())
+
+    def test_set_impurity(self):
+        test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test")
+        self.assertEqual(0.0, test.get_impurity())
+        test.set_impurity(54.7)
+        self.assertEqual(54.7, test.get_impurity())
+
+    def test_set_features(self):
+        test = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [0, 1], 0.0, "test")
+        self.assertListEqual([0, 1], test.get_features())
+        test.set_features([1, 2])
+        self.assertListEqual([1, 2], test.get_features())
 
     def test_copy_node(self):
         px = [1, 2, 3, 4]
@@ -94,3 +125,5 @@ class Snode_test(unittest.TestCase):
         self.assertEqual("test", computed._title)
         self.assertIsInstance(computed._clf, Stree)
         self.assertEqual(test._partition_column, computed._partition_column)
+        self.assertEqual(test._sample_weight, computed._sample_weight)
+        self.assertEqual(test._scaler, computed._scaler)

stree/tests/Splitter_test.py

@@ -5,7 +5,8 @@ import random
 import numpy as np
 from sklearn.svm import SVC
 from sklearn.datasets import load_wine, load_iris
-from stree import Splitter
+from stree.Splitter import Splitter
+from .utils import load_dataset, load_disc_dataset
 
 
 class Splitter_test(unittest.TestCase):
@@ -17,7 +18,7 @@ class Splitter_test(unittest.TestCase):
     def build(
         clf=SVC,
         min_samples_split=0,
-        splitter_type="random",
+        feature_select="random",
         criterion="gini",
         criteria="max_samples",
         random_state=None,
@@ -25,7 +26,7 @@ class Splitter_test(unittest.TestCase):
         return Splitter(
             clf=clf(random_state=random_state, kernel="rbf"),
             min_samples_split=min_samples_split,
-            splitter_type=splitter_type,
+            feature_select=feature_select,
             criterion=criterion,
             criteria=criteria,
             random_state=random_state,
@@ -39,20 +40,20 @@ class Splitter_test(unittest.TestCase):
         with self.assertRaises(ValueError):
             self.build(criterion="duck")
         with self.assertRaises(ValueError):
-            self.build(splitter_type="duck")
+            self.build(feature_select="duck")
         with self.assertRaises(ValueError):
             self.build(criteria="duck")
         with self.assertRaises(ValueError):
             _ = Splitter(clf=None)
-        for splitter_type in ["best", "random"]:
+        for feature_select in ["best", "random"]:
             for criterion in ["gini", "entropy"]:
                 for criteria in ["max_samples", "impurity"]:
                     tcl = self.build(
-                        splitter_type=splitter_type,
+                        feature_select=feature_select,
                         criterion=criterion,
                         criteria=criteria,
                     )
-                    self.assertEqual(splitter_type, tcl._splitter_type)
+                    self.assertEqual(feature_select, tcl._feature_select)
                     self.assertEqual(criterion, tcl._criterion)
                     self.assertEqual(criteria, tcl._criteria)
 
@@ -177,32 +178,38 @@ class Splitter_test(unittest.TestCase):
     def test_best_splitter_few_sets(self):
         X, y = load_iris(return_X_y=True)
         X = np.delete(X, 3, 1)
-        tcl = self.build(splitter_type="best", random_state=self._random_state)
+        tcl = self.build(
+            feature_select="best", random_state=self._random_state
+        )
         dataset, computed = tcl.get_subspace(X, y, max_features=2)
         self.assertListEqual([0, 2], list(computed))
         self.assertListEqual(X[:, computed].tolist(), dataset.tolist())
 
     def test_splitter_parameter(self):
         expected_values = [
-            [1, 4, 9, 12],  # best   entropy max_samples
+            [0, 6, 11, 12],  # best   entropy max_samples
-            [1, 3, 6, 10],  # best   entropy impurity
+            [0, 6, 11, 12],  # best   entropy impurity
-            [6, 8, 10, 12],  # best   gini    max_samples
+            [0, 6, 11, 12],  # best   gini    max_samples
-            [7, 8, 10, 11],  # best   gini    impurity
+            [0, 6, 11, 12],  # best   gini    impurity
             [0, 3, 8, 12],  # random entropy max_samples
-            [0, 3, 9, 11],  # random entropy impurity
+            [0, 3, 7, 12],  # random entropy impurity
-            [0, 4, 7, 12],  # random gini    max_samples
+            [1, 7, 9, 12],  # random gini    max_samples
-            [0, 2, 5, 6],  # random gini    impurity
+            [1, 5, 8, 12],  # random gini    impurity
+            [6, 9, 11, 12],  # mutual entropy max_samples
+            [6, 9, 11, 12],  # mutual entropy impurity
+            [6, 9, 11, 12],  # mutual gini    max_samples
+            [6, 9, 11, 12],  # mutual gini    impurity
         ]
         X, y = load_wine(return_X_y=True)
         rn = 0
-        for splitter_type in ["best", "random"]:
+        for feature_select in ["best", "random", "mutual"]:
             for criterion in ["entropy", "gini"]:
                 for criteria in [
                     "max_samples",
                     "impurity",
                 ]:
                     tcl = self.build(
-                        splitter_type=splitter_type,
+                        feature_select=feature_select,
                         criterion=criterion,
                         criteria=criteria,
                     )
@@ -213,12 +220,93 @@ class Splitter_test(unittest.TestCase):
                     # print(
                     #     "{},  # {:7s}{:8s}{:15s}".format(
                     #         list(computed),
-                    #         splitter_type,
+                    #         feature_select,
                     #         criterion,
                     #         criteria,
                     #     )
                     # )
-                    self.assertListEqual(expected, list(computed))
+                    self.assertListEqual(expected, sorted(list(computed)))
                     self.assertListEqual(
                         X[:, computed].tolist(), dataset.tolist()
                     )
+
+    def test_get_best_subspaces(self):
+        results = [
+            (4, [3, 4, 11, 13]),
+            (7, [1, 3, 4, 5, 11, 13, 16]),
+            (9, [1, 3, 4, 5, 7, 10, 11, 13, 16]),
+        ]
+        X, y = load_dataset(n_features=20)
+        for k, expected in results:
+            tcl = self.build(
+                feature_select="best",
+            )
+            Xs, computed = tcl.get_subspace(X, y, k)
+            self.assertListEqual(expected, list(computed))
+            self.assertListEqual(X[:, expected].tolist(), Xs.tolist())
+
+    def test_get_best_subspaces_discrete(self):
+        results = [
+            (4, [0, 3, 16, 18]),
+            (7, [0, 3, 13, 14, 16, 18, 19]),
+            (9, [0, 3, 7, 13, 14, 15, 16, 18, 19]),
+        ]
+        X, y = load_disc_dataset(n_features=20)
+        for k, expected in results:
+            tcl = self.build(
+                feature_select="best",
+            )
+            Xs, computed = tcl.get_subspace(X, y, k)
+            self.assertListEqual(expected, list(computed))
+            self.assertListEqual(X[:, expected].tolist(), Xs.tolist())
+
+    def test_get_cfs_subspaces(self):
+        results = [
+            (4, [1, 5, 9, 12]),
+            (6, [1, 5, 9, 12, 4, 2]),
+            (7, [1, 5, 9, 12, 4, 2, 3]),
+        ]
+        X, y = load_dataset(n_features=20, n_informative=7)
+        for k, expected in results:
+            tcl = self.build(feature_select="cfs")
+            Xs, computed = tcl.get_subspace(X, y, k)
+            self.assertListEqual(expected, list(computed))
+            self.assertListEqual(X[:, expected].tolist(), Xs.tolist())
+
+    def test_get_fcbf_subspaces(self):
+        results = [
+            (4, [1, 5, 9, 12]),
+            (6, [1, 5, 9, 12, 4, 2]),
+            (7, [1, 5, 9, 12, 4, 2, 16]),
+        ]
+        for rs, expected in results:
+            X, y = load_dataset(n_features=20, n_informative=7)
+            tcl = self.build(feature_select="fcbf", random_state=rs)
+            Xs, computed = tcl.get_subspace(X, y, rs)
+            self.assertListEqual(expected, list(computed))
+            self.assertListEqual(X[:, expected].tolist(), Xs.tolist())
+
+    def test_get_iwss_subspaces(self):
+        results = [
+            (4, [1, 5, 9, 12]),
+            (6, [1, 5, 9, 12, 4, 15]),
+        ]
+        for rs, expected in results:
+            X, y = load_dataset(n_features=20, n_informative=7)
+            tcl = self.build(feature_select="iwss", random_state=rs)
+            Xs, computed = tcl.get_subspace(X, y, rs)
+            self.assertListEqual(expected, list(computed))
+            self.assertListEqual(X[:, expected].tolist(), Xs.tolist())
+
+    def test_get_trandom_subspaces(self):
+        results = [
+            (4, [3, 7, 9, 12]),
+            (6, [0, 1, 2, 8, 15, 18]),
+            (7, [1, 2, 4, 8, 10, 12, 13]),
+        ]
+        for rs, expected in results:
+            X, y = load_dataset(n_features=20, n_informative=7)
+            tcl = self.build(feature_select="trandom", random_state=rs)
+            Xs, computed = tcl.get_subspace(X, y, rs)
+            self.assertListEqual(expected, list(computed))
+            self.assertListEqual(X[:, expected].tolist(), Xs.tolist())

stree/tests/Stree_test.py

@@ -7,20 +7,36 @@ from sklearn.datasets import load_iris, load_wine
 from sklearn.exceptions import ConvergenceWarning
 from sklearn.svm import LinearSVC
 
-from stree import Stree, Snode
+from stree import Stree
+from stree.Splitter import Snode
 from .utils import load_dataset
+from .._version import __version__
 
 
 class Stree_test(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         self._random_state = 1
-        self._kernels = ["linear", "rbf", "poly"]
+        self._kernels = ["liblinear", "linear", "rbf", "poly", "sigmoid"]
         super().__init__(*args, **kwargs)
 
     @classmethod
     def setUp(cls):
         os.environ["TESTING"] = "1"
 
+    def test_valid_kernels(self):
+        X, y = load_dataset()
+        for kernel in self._kernels:
+            clf = Stree(kernel=kernel, multiclass_strategy="ovr")
+            clf.fit(X, y)
+            self.assertIsNotNone(clf.tree_)
+
+    def test_bogus_kernel(self):
+        kernel = "other"
+        X, y = load_dataset()
+        clf = Stree(kernel=kernel)
+        with self.assertRaises(ValueError):
+            clf.fit(X, y)
+
     def _check_tree(self, node: Snode):
         """Check recursively that the nodes that are not leaves have the
         correct number of labels and its sons have the right number of elements
@@ -40,14 +56,19 @@ class Stree_test(unittest.TestCase):
         # i.e. The partition algorithm didn't forget any sample
         self.assertEqual(node._y.shape[0], y_down.shape[0] + y_up.shape[0])
         unique_y, count_y = np.unique(node._y, return_counts=True)
-        _, count_d = np.unique(y_down, return_counts=True)
+        labels_d, count_d = np.unique(y_down, return_counts=True)
-        _, count_u = np.unique(y_up, return_counts=True)
+        labels_u, count_u = np.unique(y_up, return_counts=True)
+        dict_d = {label: count_d[i] for i, label in enumerate(labels_d)}
+        dict_u = {label: count_u[i] for i, label in enumerate(labels_u)}
         #
         for i in unique_y:
-            number_up = count_u[i]
             try:
-                number_down = count_d[i]
+                number_up = dict_u[i]
-            except IndexError:
+            except KeyError:
+                number_up = 0
+            try:
+                number_down = dict_d[i]
+            except KeyError:
                 number_down = 0
             self.assertEqual(count_y[i], number_down + number_up)
         # Is the partition made the same as the prediction?
@@ -62,14 +83,22 @@ class Stree_test(unittest.TestCase):
         """Check if the tree is built the same way as predictions of models"""
         warnings.filterwarnings("ignore")
         for kernel in self._kernels:
-            clf = Stree(kernel=kernel, random_state=self._random_state)
+            clf = Stree(
+                kernel="sigmoid",
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
+                random_state=self._random_state,
+            )
             clf.fit(*load_dataset(self._random_state))
             self._check_tree(clf.tree_)
 
     def test_single_prediction(self):
         X, y = load_dataset(self._random_state)
         for kernel in self._kernels:
-            clf = Stree(kernel=kernel, random_state=self._random_state)
+            clf = Stree(
+                kernel=kernel,
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
+                random_state=self._random_state,
+            )
             yp = clf.fit(X, y).predict((X[0, :].reshape(-1, X.shape[1])))
             self.assertEqual(yp[0], y[0])
 
@@ -77,18 +106,58 @@ class Stree_test(unittest.TestCase):
         # First 27 elements the predictions are the same as the truth
         num = 27
         X, y = load_dataset(self._random_state)
-        for kernel in self._kernels:
+        for kernel in ["liblinear", "linear", "rbf", "poly"]:
-            clf = Stree(kernel=kernel, random_state=self._random_state)
+            clf = Stree(
+                kernel=kernel,
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
+                random_state=self._random_state,
+            )
             yp = clf.fit(X, y).predict(X[:num, :])
             self.assertListEqual(y[:num].tolist(), yp.tolist())
 
+    def test_multiple_predict_proba(self):
+        expected = {
+            "liblinear": {
+                0: [0.02401129943502825, 0.9759887005649718],
+                17: [0.9282970550576184, 0.07170294494238157],
+            },
+            "linear": {
+                0: [0.029329608938547486, 0.9706703910614525],
+                17: [0.9298469387755102, 0.07015306122448979],
+            },
+            "rbf": {
+                0: [0.023448275862068966, 0.976551724137931],
+                17: [0.9458064516129032, 0.05419354838709677],
+            },
+            "poly": {
+                0: [0.01601164483260553, 0.9839883551673945],
+                17: [0.9089790897908979, 0.0910209102091021],
+            },
+        }
+        indices = [0, 17]
+        X, y = load_dataset(self._random_state)
+        for kernel in ["liblinear", "linear", "rbf", "poly"]:
+            clf = Stree(
+                kernel=kernel,
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
+                random_state=self._random_state,
+            )
+            yp = clf.fit(X, y).predict_proba(X)
+            for index in indices:
+                for exp, comp in zip(expected[kernel][index], yp[index]):
+                    self.assertAlmostEqual(exp, comp)
+
     def test_single_vs_multiple_prediction(self):
         """Check if predicting sample by sample gives the same result as
         predicting all samples at once
         """
         X, y = load_dataset(self._random_state)
         for kernel in self._kernels:
-            clf = Stree(kernel=kernel, random_state=self._random_state)
+            clf = Stree(
+                kernel=kernel,
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
+                random_state=self._random_state,
+            )
             clf.fit(X, y)
             # Compute prediction line by line
             yp_line = np.array([], dtype=int)
@@ -103,26 +172,30 @@ class Stree_test(unittest.TestCase):
     def test_iterator_and_str(self):
         """Check preorder iterator"""
         expected = [
-            "root feaures=(0, 1, 2) impurity=1.0000 counts=(array([0, 1]), arr"
+            "root feaures=(0, 1, 2) impurity=1.0000 counts=(array([0, 1]), "
-            "ay([750, 750]))",
+            "array([750, 750]))",
-            "root - Down, <cgaf> - Leaf class=0 belief= 0.928297 impurity=0.37"
+            "root - Down(2), <cgaf> - Leaf class=0 belief= 0.928297 impurity="
-            "22 counts=(array([0, 1]), array([725,  56]))",
+            "0.3722 counts=(array([0, 1]), array([725,  56]))",
-            "root - Up feaures=(0, 1, 2) impurity=0.2178 counts=(array([0, 1])"
+            "root - Up(2) feaures=(0, 1, 2) impurity=0.2178 counts=(array([0, "
-            ", array([ 25, 694]))",
+            "1]), array([ 25, 694]))",
-            "root - Up - Down feaures=(0, 1, 2) impurity=0.8454 counts=(array("
+            "root - Up(2) - Down(3) feaures=(0, 1, 2) impurity=0.8454 counts="
-            "[0, 1]), array([8, 3]))",
+            "(array([0, 1]), array([8, 3]))",
-            "root - Up - Down - Down, <pure> - Leaf class=0 belief= 1.000000 i"
+            "root - Up(2) - Down(3) - Down(4), <pure> - Leaf class=0 belief= "
-            "mpurity=0.0000 counts=(array([0]), array([7]))",
+            "1.000000 impurity=0.0000 counts=(array([0]), array([7]))",
-            "root - Up - Down - Up, <cgaf> - Leaf class=1 belief= 0.750000 imp"
+            "root - Up(2) - Down(3) - Up(4), <cgaf> - Leaf class=1 belief= "
-            "urity=0.8113 counts=(array([0, 1]), array([1, 3]))",
+            "0.750000 impurity=0.8113 counts=(array([0, 1]), array([1, 3]))",
-            "root - Up - Up, <cgaf> - Leaf class=1 belief= 0.975989 impurity=0"
+            "root - Up(2) - Up(3), <cgaf> - Leaf class=1 belief= 0.975989 "
-            ".1634 counts=(array([0, 1]), array([ 17, 691]))",
+            "impurity=0.1634 counts=(array([0, 1]), array([ 17, 691]))",
         ]
         computed = []
         expected_string = ""
-        clf = Stree(kernel="linear", random_state=self._random_state)
+        clf = Stree(
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+            random_state=self._random_state,
+        )
         clf.fit(*load_dataset(self._random_state))
-        for node in clf:
+        for node in iter(clf):
             computed.append(str(node))
             expected_string += str(node) + "\n"
         self.assertListEqual(expected, computed)
@@ -158,7 +231,12 @@ class Stree_test(unittest.TestCase):
     def test_check_max_depth(self):
         depths = (3, 4)
         for depth in depths:
-            tcl = Stree(random_state=self._random_state, max_depth=depth)
+            tcl = Stree(
+                kernel="liblinear",
+                multiclass_strategy="ovr",
+                random_state=self._random_state,
+                max_depth=depth,
+            )
             tcl.fit(*load_dataset(self._random_state))
             self.assertEqual(depth, tcl.depth_)
 
@@ -179,7 +257,7 @@ class Stree_test(unittest.TestCase):
         for kernel in self._kernels:
             clf = Stree(
                 kernel=kernel,
-                split_criteria="max_samples",
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
                 random_state=self._random_state,
             )
             px = [[1, 2], [5, 6], [9, 10]]
@@ -190,26 +268,36 @@ class Stree_test(unittest.TestCase):
             self.assertListEqual(py, clf.classes_.tolist())
 
     def test_muticlass_dataset(self):
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        warnings.filterwarnings("ignore", category=RuntimeWarning)
         datasets = {
             "Synt": load_dataset(random_state=self._random_state, n_classes=3),
             "Iris": load_wine(return_X_y=True),
         }
         outcomes = {
             "Synt": {
-                "max_samples linear": 0.9606666666666667,
+                "max_samples liblinear": 0.9493333333333334,
-                "max_samples rbf": 0.7133333333333334,
+                "max_samples linear": 0.9426666666666667,
-                "max_samples poly": 0.49066666666666664,
+                "max_samples rbf": 0.9606666666666667,
-                "impurity linear": 0.9606666666666667,
+                "max_samples poly": 0.9373333333333334,
-                "impurity rbf": 0.7133333333333334,
+                "max_samples sigmoid": 0.824,
-                "impurity poly": 0.49066666666666664,
+                "impurity liblinear": 0.9493333333333334,
+                "impurity linear": 0.9426666666666667,
+                "impurity rbf": 0.9606666666666667,
+                "impurity poly": 0.9373333333333334,
+                "impurity sigmoid": 0.824,
             },
             "Iris": {
+                "max_samples liblinear": 0.9550561797752809,
                 "max_samples linear": 1.0,
-                "max_samples rbf": 0.6910112359550562,
+                "max_samples rbf": 0.6685393258426966,
-                "max_samples poly": 0.6966292134831461,
+                "max_samples poly": 0.6853932584269663,
-                "impurity linear": 1,
+                "max_samples sigmoid": 0.6404494382022472,
-                "impurity rbf": 0.6910112359550562,
+                "impurity liblinear": 0.9550561797752809,
-                "impurity poly": 0.6966292134831461,
+                "impurity linear": 1.0,
+                "impurity rbf": 0.6685393258426966,
+                "impurity poly": 0.6853932584269663,
+                "impurity sigmoid": 0.6404494382022472,
             },
         }
 
@@ -218,18 +306,22 @@ class Stree_test(unittest.TestCase):
             for criteria in ["max_samples", "impurity"]:
                 for kernel in self._kernels:
                     clf = Stree(
-                        C=55,
+                        max_iter=1e4,
-                        max_iter=1e5,
+                        multiclass_strategy="ovr"
+                        if kernel == "liblinear"
+                        else "ovo",
                         kernel=kernel,
                         random_state=self._random_state,
                     )
                     clf.fit(px, py)
                     outcome = outcomes[name][f"{criteria} {kernel}"]
-                    # print(
+                    # print(f'"{criteria} {kernel}": {clf.score(px, py)},')
-                    #     f"{name} {criteria} {kernel} {outcome} {clf.score(px"
+                    self.assertAlmostEqual(
-                    #     ", py)}"
+                        outcome,
-                    # )
+                        clf.score(px, py),
-                    self.assertAlmostEqual(outcome, clf.score(px, py))
+                        5,
+                        f"{name} - {criteria} - {kernel}",
+                    )
 
     def test_max_features(self):
         n_features = 16
@@ -254,6 +346,12 @@ class Stree_test(unittest.TestCase):
             with self.assertRaises(ValueError):
                 _ = clf._initialize_max_features()
 
+    def test_wrong_max_features(self):
+        X, y = load_dataset(n_features=15)
+        clf = Stree(max_features=16)
+        with self.assertRaises(ValueError):
+            clf.fit(X, y)
+
     def test_get_subspaces(self):
         dataset = np.random.random((10, 16))
         y = np.random.randint(0, 2, 10)
@@ -291,17 +389,20 @@ class Stree_test(unittest.TestCase):
             clf.predict(X[:, :3])
 
     # Tests of score
-
     def test_score_binary(self):
+        """Check score for binary classification."""
         X, y = load_dataset(self._random_state)
         accuracies = [
             0.9506666666666667,
+            0.9493333333333334,
             0.9606666666666667,
             0.9433333333333334,
+            0.9153333333333333,
         ]
         for kernel, accuracy_expected in zip(self._kernels, accuracies):
             clf = Stree(
                 random_state=self._random_state,
+                multiclass_strategy="ovr" if kernel == "liblinear" else "ovo",
                 kernel=kernel,
             )
             clf.fit(X, y)
@@ -312,12 +413,19 @@ class Stree_test(unittest.TestCase):
             self.assertAlmostEqual(accuracy_expected, accuracy_score)
 
     def test_score_max_features(self):
+        """Check score using max_features."""
         X, y = load_dataset(self._random_state)
-        clf = Stree(random_state=self._random_state, max_features=2)
+        clf = Stree(
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+            random_state=self._random_state,
+            max_features=2,
+        )
         clf.fit(X, y)
-        self.assertAlmostEqual(0.9246666666666666, clf.score(X, y))
+        self.assertAlmostEqual(0.9453333333333334, clf.score(X, y))
 
     def test_bogus_splitter_parameter(self):
+        """Check that bogus splitter parameter raises exception."""
         clf = Stree(splitter="duck")
         with self.assertRaises(ValueError):
             clf.fit(*load_dataset())
@@ -325,7 +433,9 @@ class Stree_test(unittest.TestCase):
     def test_multiclass_classifier_integrity(self):
         """Checks if the multiclass operation is done right"""
         X, y = load_iris(return_X_y=True)
-        clf = Stree(random_state=0)
+        clf = Stree(
+            kernel="liblinear", multiclass_strategy="ovr", random_state=0
+        )
         clf.fit(X, y)
         score = clf.score(X, y)
         # Check accuracy of the whole model
@@ -371,6 +481,7 @@ class Stree_test(unittest.TestCase):
         self.assertListEqual([47], resdn[1].tolist())
 
     def test_score_multiclass_rbf(self):
+        """Test score for multiclass classification with rbf kernel."""
         X, y = load_dataset(
             random_state=self._random_state,
             n_classes=3,
@@ -378,11 +489,17 @@ class Stree_test(unittest.TestCase):
             n_samples=500,
         )
         clf = Stree(kernel="rbf", random_state=self._random_state)
-        self.assertEqual(0.824, clf.fit(X, y).score(X, y))
+        clf2 = Stree(
+            kernel="rbf", random_state=self._random_state, normalize=True
+        )
+        self.assertEqual(0.966, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.964, clf2.fit(X, y).score(X, y))
         X, y = load_wine(return_X_y=True)
-        self.assertEqual(0.6741573033707865, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.6685393258426966, clf.fit(X, y).score(X, y))
+        self.assertEqual(1.0, clf2.fit(X, y).score(X, y))
 
     def test_score_multiclass_poly(self):
+        """Test score for multiclass classification with poly kernel."""
         X, y = load_dataset(
             random_state=self._random_state,
             n_classes=3,
@@ -392,28 +509,100 @@ class Stree_test(unittest.TestCase):
         clf = Stree(
             kernel="poly", random_state=self._random_state, C=10, degree=5
         )
-        self.assertEqual(0.786, clf.fit(X, y).score(X, y))
+        clf2 = Stree(
+            kernel="poly",
+            random_state=self._random_state,
+            normalize=True,
+        )
+        self.assertEqual(0.946, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.972, clf2.fit(X, y).score(X, y))
         X, y = load_wine(return_X_y=True)
-        self.assertEqual(0.702247191011236, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.7808988764044944, clf.fit(X, y).score(X, y))
+        self.assertEqual(1.0, clf2.fit(X, y).score(X, y))
+
+    def test_score_multiclass_liblinear(self):
+        """Test score for multiclass classification with liblinear kernel."""
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=500,
+        )
+        clf = Stree(
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+            random_state=self._random_state,
+            C=10,
+        )
+        clf2 = Stree(
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+            random_state=self._random_state,
+            normalize=True,
+        )
+        self.assertEqual(0.968, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.97, clf2.fit(X, y).score(X, y))
+        X, y = load_wine(return_X_y=True)
+        self.assertEqual(1.0, clf.fit(X, y).score(X, y))
+        self.assertEqual(1.0, clf2.fit(X, y).score(X, y))
+
+    def test_score_multiclass_sigmoid(self):
+        """Test score for multiclass classification with sigmoid kernel."""
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=500,
+        )
+        clf = Stree(kernel="sigmoid", random_state=self._random_state, C=10)
+        clf2 = Stree(
+            kernel="sigmoid",
+            random_state=self._random_state,
+            normalize=True,
+            C=10,
+        )
+        self.assertEqual(0.796, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.952, clf2.fit(X, y).score(X, y))
+        X, y = load_wine(return_X_y=True)
+        self.assertEqual(0.6910112359550562, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.9662921348314607, clf2.fit(X, y).score(X, y))
 
     def test_score_multiclass_linear(self):
+        """Test score for multiclass classification with linear kernel."""
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        warnings.filterwarnings("ignore", category=RuntimeWarning)
         X, y = load_dataset(
             random_state=self._random_state,
             n_classes=3,
             n_features=5,
             n_samples=1500,
         )
-        clf = Stree(kernel="linear", random_state=self._random_state)
+        clf = Stree(
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+            random_state=self._random_state,
+        )
         self.assertEqual(0.9533333333333334, clf.fit(X, y).score(X, y))
+        # Check with context based standardization
+        clf2 = Stree(
+            kernel="liblinear",
+            multiclass_strategy="ovr",
+            random_state=self._random_state,
+            normalize=True,
+        )
+        self.assertEqual(0.9526666666666667, clf2.fit(X, y).score(X, y))
         X, y = load_wine(return_X_y=True)
-        self.assertEqual(0.9550561797752809, clf.fit(X, y).score(X, y))
+        self.assertEqual(0.9831460674157303, clf.fit(X, y).score(X, y))
+        self.assertEqual(1.0, clf2.fit(X, y).score(X, y))
 
     def test_zero_all_sample_weights(self):
+        """Test exception raises when all sample weights are zero."""
         X, y = load_dataset(self._random_state)
         with self.assertRaises(ValueError):
             Stree().fit(X, y, np.zeros(len(y)))
 
     def test_mask_samples_weighted_zero(self):
+        """Check that the weighted zero samples are masked."""
         X = np.array(
             [
                 [1, 1],
@@ -428,7 +617,7 @@ class Stree_test(unittest.TestCase):
             ]
         )
         y = np.array([1, 1, 1, 2, 2, 2, 5, 5, 5])
-        yw = np.array([1, 1, 1, 5, 5, 5, 5, 5, 5])
+        yw = np.array([1, 1, 1, 1, 1, 1, 5, 5, 5])
         w = [1, 1, 1, 0, 0, 0, 1, 1, 1]
         model1 = Stree().fit(X, y)
         model2 = Stree().fit(X, y, w)
@@ -439,3 +628,132 @@ class Stree_test(unittest.TestCase):
         self.assertEqual(model1.score(X, y), 1)
         self.assertAlmostEqual(model2.score(X, y), 0.66666667)
         self.assertEqual(model2.score(X, y, w), 1)
+
+    def test_depth(self):
+        """Check depth of the tree."""
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=1500,
+        )
+        clf = Stree(random_state=self._random_state)
+        clf.fit(X, y)
+        self.assertEqual(6, clf.depth_)
+        X, y = load_wine(return_X_y=True)
+        clf = Stree(random_state=self._random_state)
+        clf.fit(X, y)
+        self.assertEqual(4, clf.depth_)
+
+    def test_nodes_leaves(self):
+        """Check number of nodes and leaves."""
+        X, y = load_dataset(
+            random_state=self._random_state,
+            n_classes=3,
+            n_features=5,
+            n_samples=1500,
+        )
+        clf = Stree(random_state=self._random_state)
+        clf.fit(X, y)
+        nodes, leaves = clf.nodes_leaves()
+        self.assertEqual(31, nodes)
+        self.assertEqual(16, leaves)
+        X, y = load_wine(return_X_y=True)
+        clf = Stree(random_state=self._random_state)
+        clf.fit(X, y)
+        nodes, leaves = clf.nodes_leaves()
+        self.assertEqual(11, nodes)
+        self.assertEqual(6, leaves)
+
+    def test_nodes_leaves_artificial(self):
+        """Check leaves of artificial dataset."""
+        n1 = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test1")
+        n2 = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test2")
+        n3 = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test3")
+        n4 = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test4")
+        n5 = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test5")
+        n6 = Snode(None, [1, 2, 3, 4], [1, 0, 1, 1], [], 0.0, "test6")
+        n1.set_up(n2)
+        n2.set_up(n3)
+        n2.set_down(n4)
+        n3.set_up(n5)
+        n4.set_down(n6)
+        clf = Stree(random_state=self._random_state)
+        clf.tree_ = n1
+        nodes, leaves = clf.nodes_leaves()
+        self.assertEqual(6, nodes)
+        self.assertEqual(2, leaves)
+
+    def test_bogus_multiclass_strategy(self):
+        """Check invalid multiclass strategy."""
+        clf = Stree(multiclass_strategy="other")
+        X, y = load_wine(return_X_y=True)
+        with self.assertRaises(ValueError):
+            clf.fit(X, y)
+
+    def test_multiclass_strategy(self):
+        """Check multiclass strategy."""
+        X, y = load_wine(return_X_y=True)
+        clf_o = Stree(multiclass_strategy="ovo")
+        clf_r = Stree(multiclass_strategy="ovr")
+        score_o = clf_o.fit(X, y).score(X, y)
+        score_r = clf_r.fit(X, y).score(X, y)
+        self.assertEqual(1.0, score_o)
+        self.assertEqual(0.9269662921348315, score_r)
+
+    def test_incompatible_hyperparameters(self):
+        """Check incompatible hyperparameters."""
+        X, y = load_wine(return_X_y=True)
+        clf = Stree(kernel="liblinear", multiclass_strategy="ovo")
+        with self.assertRaises(ValueError):
+            clf.fit(X, y)
+        clf = Stree(multiclass_strategy="ovo", split_criteria="max_samples")
+        with self.assertRaises(ValueError):
+            clf.fit(X, y)
+
+    def test_version(self):
+        """Check STree version."""
+        clf = Stree()
+        self.assertEqual(__version__, clf.version())
+
+    def test_graph(self):
+        """Check graphviz representation of the tree."""
+        X, y = load_wine(return_X_y=True)
+        clf = Stree(random_state=self._random_state)
+
+        expected_head = (
+            "digraph STree {\nlabel=<STree >\nfontsize=30\n"
+            "fontcolor=blue\nlabelloc=t\n"
+        )
+        expected_tail = (
+            ' [shape=box style=filled label="class=1 impurity=0.000 '
+            'counts=[0 1 0]"];\n}\n'
+        )
+        self.assertEqual(clf.graph(), expected_head + "}\n")
+        clf.fit(X, y)
+        computed = clf.graph()
+        computed_head = computed[: len(expected_head)]
+        num = -len(expected_tail)
+        computed_tail = computed[num:]
+        self.assertEqual(computed_head, expected_head)
+        self.assertEqual(computed_tail, expected_tail)
+
+    def test_graph_title(self):
+        X, y = load_wine(return_X_y=True)
+        clf = Stree(random_state=self._random_state)
+        expected_head = (
+            "digraph STree {\nlabel=<STree Sample title>\nfontsize=30\n"
+            "fontcolor=blue\nlabelloc=t\n"
+        )
+        expected_tail = (
+            ' [shape=box style=filled label="class=1 impurity=0.000 '
+            'counts=[0 1 0]"];\n}\n'
+        )
+        self.assertEqual(clf.graph("Sample title"), expected_head + "}\n")
+        clf.fit(X, y)
+        computed = clf.graph("Sample title")
+        computed_head = computed[: len(expected_head)]
+        num = -len(expected_tail)
+        computed_tail = computed[num:]
+        self.assertEqual(computed_head, expected_head)
+        self.assertEqual(computed_tail, expected_tail)

stree/tests/utils.py

@@ -1,11 +1,14 @@
 from sklearn.datasets import make_classification
+import numpy as np
 
 
-def load_dataset(random_state=0, n_classes=2, n_features=3, n_samples=1500):
+def load_dataset(
+    random_state=0, n_classes=2, n_features=3, n_samples=1500, n_informative=3
+):
     X, y = make_classification(
         n_samples=n_samples,
         n_features=n_features,
-        n_informative=3,
+        n_informative=n_informative,
         n_redundant=0,
         n_repeated=0,
         n_classes=n_classes,
@@ -15,3 +18,12 @@ def load_dataset(random_state=0, n_classes=2, n_features=3, n_samples=1500):
         random_state=random_state,
     )
     return X, y
+
+
+def load_disc_dataset(
+    random_state=0, n_classes=2, n_features=3, n_samples=1500
+):
+    np.random.seed(random_state)
+    X = np.random.randint(1, 17, size=(n_samples, n_features)).astype(float)
+    y = np.random.randint(low=0, high=n_classes, size=(n_samples), dtype=int)
+    return X, y