Update README.md

Update README.md

.env.dist

@@ -4,3 +4,5 @@ n_folds=5
 model=ODTE
 stratified=0
 source_data=Tanveer
+seeds=[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]
+discretize=0

.flake8

@@ -1,2 +1,3 @@
 [flake8]
 exclude = .git,__init__.py
+ignore = E203, W503

.github/workflows/build.yml

@@ -8,25 +8,22 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Get project version
-        id: step_one
-        run: |
-          version=$(git describe --tags --abbrev=0)
-          echo "project_version=$version" >> $GITHUB_ENV
+      - run: echo "project_version=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
       - uses: sonarsource/sonarqube-scan-action@master
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
-          with:
-            args: >
-              -Dsonar.projectVersion=${{ env.project_version }}
-              -Dsonar.python.version=3.10
+        with:
+          args: >
+            -Dsonar.projectVersion=${{ env.project_version }}
+            -Dsonar.python.version=3.10
         # If you wish to fail your job when the Quality Gate is red, uncomment the
         # following lines. This would typically be used to fail a deployment.
-      - uses: sonarsource/sonarqube-quality-gate-action@master
-        timeout-minutes: 5
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      #- uses: sonarsource/sonarqube-quality-gate-action@master
+      #  timeout-minutes: 5
+      #  env:
+      #    SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      #    SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}

.github/workflows/main.yml

@@ -12,13 +12,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest]
+        os: [ubuntu-latest]
         python: ["3.10"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       # Make dot command available in the environment
@@ -43,16 +43,17 @@ jobs:
           pip install -q --upgrade pip
           pip install -q -r requirements.txt
           pip install -q --upgrade codecov coverage black flake8
+          git clone https://github.com/Doctorado-ML/bayesclass.git
       - name: Lint
         run: |
           black --check --diff benchmark
-          flake8 --count benchmark
+          flake8 --count benchmark --ignore=E203,W503
       - name: Tests
         run: |
           coverage run -m unittest -v benchmark.tests
           coverage xml
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml

README.md

@@ -1,12 +1,9 @@
-[![CI](https://github.com/Doctorado-ML/benchmark/actions/workflows/main.yml/badge.svg)](https://github.com/Doctorado-ML/benchmark/actions/workflows/main.yml)
 [![codecov](https://codecov.io/gh/Doctorado-ML/benchmark/branch/main/graph/badge.svg?token=ZRP937NDSG)](https://codecov.io/gh/Doctorado-ML/benchmark)
-[![Quality Gate Status](https://haystack.rmontanana.es:25000/api/project_badges/measure?project=benchmark&metric=alert_status&token=336a6e501988888543c3153baa91bad4b9914dd2)](http://haystack.local:25000/dashboard?id=benchmark)
-[![Technical Debt](https://haystack.rmontanana.es:25000/api/project_badges/measure?project=benchmark&metric=sqale_index&token=336a6e501988888543c3153baa91bad4b9914dd2)](http://haystack.local:25000/dashboard?id=benchmark)
 ![https://img.shields.io/badge/python-3.8%2B-blue](https://img.shields.io/badge/python-3.8%2B-brightgreen)
 
 # benchmark
 
-Benchmarking models
+Benchmarking Python models
 
 ## Experimentation
 
@@ -34,7 +31,7 @@ be_report -b STree
 
 ```python
 # Datasets list
-be_report
+be_report datasets
 # Report of given experiment
 be_report -f results/results_STree_iMac27_2021-09-22_17:13:02.json
 # Report of given experiment building excel file and compare with best results

benchmark/Arguments.py

@@ -1,6 +1,6 @@
 import sys
 import argparse
-from .Experiments import Models
+from .Models import Models
 from .Utils import Files, NO_ENV
 
 ALL_METRICS = (
@@ -13,21 +13,27 @@ ALL_METRICS = (
 
 
 class EnvData:
-    @staticmethod
-    def load():
-        args = {}
+    def __init__(self):
+        self.args = {}
+
+    def load(self):
         try:
             with open(Files.dot_env) as f:
                 for line in f.read().splitlines():
                     if line == "" or line.startswith("#"):
                         continue
                     key, value = line.split("=")
-                    args[key] = value
+                    self.args[key] = value
         except FileNotFoundError:
             print(NO_ENV, file=sys.stderr)
             exit(1)
         else:
-            return args
+            return self.args
+
+    def save(self):
+        with open(Files.dot_env, "w") as f:
+            for key, value in self.args.items():
+                f.write(f"{key}={value}\n")
 
 
 class EnvDefault(argparse.Action):
@@ -35,7 +41,8 @@ class EnvDefault(argparse.Action):
     def __init__(
         self, envvar, required=True, default=None, mandatory=False, **kwargs
     ):
-        self._args = EnvData.load()
+        self._args = EnvData().load()
+        self._overrides = {}
         if required and not mandatory:
             default = self._args[envvar]
             required = False
@@ -47,24 +54,27 @@ class EnvDefault(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
-class Arguments:
-    def __init__(self):
-        self.ap = argparse.ArgumentParser()
+class Arguments(argparse.ArgumentParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         models_data = Models.define_models(random_state=0)
+        self._overrides = {}
+        self._subparser = None
         self.parameters = {
-            "best": [
-                ("-b", "--best"),
+            "best_paramfile": [
+                ("-b", "--best_paramfile"),
                 {
-                    "type": str,
+                    "action": "store_true",
                     "required": False,
-                    "help": "best results of models",
+                    "default": False,
+                    "help": "Use best hyperparams file?",
                 },
             ],
             "color": [
                 ("-c", "--color"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "use colors for the tree",
                 },
@@ -72,8 +82,9 @@ class Arguments:
             "compare": [
                 ("-c", "--compare"),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "required": False,
+                    "default": False,
                     "help": "Compare accuracy with best results",
                 },
             ],
@@ -81,45 +92,57 @@ class Arguments:
                 ("-d", "--dataset"),
                 {
                     "type": str,
+                    "envvar": "dataset",  # for compatiblity with EnvDefault
+                    "action": EnvDefault,
                     "required": False,
                     "help": "dataset to work with",
                 },
             ],
+            "discretize": [
+                ("--discretize",),
+                {
+                    "action": EnvDefault,
+                    "envvar": "discretize",
+                    "required": True,
+                    "help": "Discretize dataset",
+                    "const": "1",
+                    "nargs": "?",
+                },
+            ],
             "excel": [
                 ("-x", "--excel"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "Generate Excel File",
                 },
             ],
-            "file": [
-                ("-f", "--file"),
-                {"type": str, "required": False, "help": "Result file"},
-            ],
-            "grid": [
-                ("-g", "--grid"),
+            "fit_features": [
+                ("--fit_features",),
                 {
-                    "type": str,
-                    "required": False,
-                    "help": "grid results of model",
+                    "action": EnvDefault,
+                    "envvar": "fit_features",
+                    "required": True,
+                    "help": "Include features in fit call",
+                    "const": "1",
+                    "nargs": "?",
                 },
             ],
             "grid_paramfile": [
                 ("-g", "--grid_paramfile"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
-                    "help": "Use best hyperparams file?",
+                    "help": "Use grid output hyperparams file?",
                 },
             ],
             "hidden": [
                 ("--hidden",),
                 {
-                    "type": str,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "Show hidden results",
                 },
@@ -128,6 +151,15 @@ class Arguments:
                 ("-p", "--hyperparameters"),
                 {"type": str, "required": False, "default": "{}"},
             ],
+            "ignore_nan": [
+                ("--ignore-nan",),
+                {
+                    "default": False,
+                    "action": "store_true",
+                    "required": False,
+                    "help": "Ignore nan results",
+                },
+            ],
             "key": [
                 ("-k", "--key"),
                 {
@@ -140,8 +172,8 @@ class Arguments:
             "lose": [
                 ("-l", "--lose"),
                 {
-                    "type": bool,
                     "default": False,
+                    "action": "store_true",
                     "required": False,
                     "help": "show lose results",
                 },
@@ -178,9 +210,10 @@ class Arguments:
             "nan": [
                 ("--nan",),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "required": False,
-                    "help": "Move nan results to hidden folder",
+                    "default": False,
+                    "help": "List nan results to hidden folder",
                 },
             ],
             "number": [
@@ -202,13 +235,17 @@ class Arguments:
                     "help": "number of folds",
                 },
             ],
-            "paramfile": [
-                ("-f", "--paramfile"),
+            "output": [
+                ("-o", "--output"),
                 {
-                    "type": bool,
+                    "type": str,
+                    "default": "local",
+                    "choices": ["local", "docker"],
                     "required": False,
-                    "default": False,
-                    "help": "Use best hyperparams file?",
+                    "help": (
+                        "in be_flask tells if it is running in local or "
+                        "in docker {local, docker}"
+                    ),
                 },
             ],
             "platform": [
@@ -224,7 +261,7 @@ class Arguments:
             "quiet": [
                 ("-q", "--quiet"),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "required": False,
                     "default": False,
                 },
@@ -232,7 +269,7 @@ class Arguments:
             "report": [
                 ("-r", "--report"),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "default": False,
                     "required": False,
                     "help": "Report results",
@@ -250,23 +287,29 @@ class Arguments:
             ],
             "sql": [
                 ("-q", "--sql"),
-                {"type": bool, "required": False, "help": "Generate SQL File"},
+                {
+                    "required": False,
+                    "action": "store_true",
+                    "default": False,
+                    "help": "Generate SQL File",
+                },
             ],
             "stratified": [
                 ("-t", "--stratified"),
                 {
                     "action": EnvDefault,
                     "envvar": "stratified",
-                    "type": str,
                     "required": True,
                     "help": "Stratified",
+                    "const": "1",
+                    "nargs": "?",
                 },
             ],
             "tex_output": [
                 ("-t", "--tex-output"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "Generate Tex file with the table",
                 },
@@ -278,8 +321,8 @@ class Arguments:
             "win": [
                 ("-w", "--win"),
                 {
-                    "type": bool,
                     "default": False,
+                    "action": "store_true",
                     "required": False,
                     "help": "show win results",
                 },
@@ -287,12 +330,43 @@ class Arguments:
         }
 
     def xset(self, *arg_name, **kwargs):
-        names, default = self.parameters[arg_name[0]]
-        self.ap.add_argument(
+        names, parameters = self.parameters[arg_name[0]]
+        if "overrides" in kwargs:
+            self._overrides[names[0]] = (kwargs["overrides"], kwargs["const"])
+            del kwargs["overrides"]
+        self.add_argument(
             *names,
-            **{**default, **kwargs},
+            **{**parameters, **kwargs},
         )
         return self
 
+    def add_subparser(
+        self, dest="subcommand", help_text="help for subcommand"
+    ):
+        self._subparser = self.add_subparsers(dest=dest, help=help_text)
+
+    def add_subparsers_options(self, subparser, arguments):
+        command, help_text = subparser
+        parser = self._subparser.add_parser(command, help=help_text)
+        for name, args in arguments:
+            try:
+                names, parameters = self.parameters[name]
+            except KeyError:
+                names = (name,)
+                parameters = {}
+            # Order of args is important
+            parser.add_argument(*names, **{**args, **parameters})
+
+    def add_exclusive(self, hyperparameters, required=False):
+        group = self.add_mutually_exclusive_group(required=required)
+        for name in hyperparameters:
+            names, parameters = self.parameters[name]
+            group.add_argument(*names, **parameters)
+
     def parse(self, args=None):
-        return self.ap.parse_args(args)
+        for key, (dest_key, value) in self._overrides.items():
+            if args is None:
+                args = sys.argv[1:]
+            if key in args:
+                args.extend((f"--{dest_key}", value))
+        return super().parse_args(args)

benchmark/Datasets.py

@@ -0,0 +1,240 @@
+import os
+from types import SimpleNamespace
+import pandas as pd
+import numpy as np
+import json
+from scipy.io import arff
+from .Utils import Files
+from .Arguments import EnvData
+from fimdlp.mdlp import FImdlp
+
+
+class Diterator:
+    def __init__(self, data):
+        self._stack = data.copy()
+
+    def __next__(self):
+        if len(self._stack) == 0:
+            raise StopIteration()
+        return self._stack.pop(0)
+
+
+class DatasetsArff:
+    @staticmethod
+    def dataset_names(name):
+        return f"{name}.arff"
+
+    @staticmethod
+    def folder():
+        return "datasets"
+
+    @staticmethod
+    def get_range_features(X, c_features):
+        if c_features.strip() == "all":
+            return list(range(X.shape[1]))
+        if c_features.strip() == "none":
+            return []
+        return json.loads(c_features)
+
+    def load(self, name, class_name):
+        file_name = os.path.join(self.folder(), self.dataset_names(name))
+        data = arff.loadarff(file_name)
+        df = pd.DataFrame(data[0])
+        df.dropna(axis=0, how="any", inplace=True)
+        self.dataset = df
+        X = df.drop(class_name, axis=1)
+        self.features = X.columns.to_list()
+        self.class_name = class_name
+        y, _ = pd.factorize(df[class_name])
+        X = X.to_numpy()
+        return X, y
+
+
+class DatasetsTanveer:
+    @staticmethod
+    def dataset_names(name):
+        return f"{name}_R.dat"
+
+    @staticmethod
+    def folder():
+        return "data"
+
+    @staticmethod
+    def get_range_features(X, name):
+        return []
+
+    def load(self, name, *args):
+        file_name = os.path.join(self.folder(), self.dataset_names(name))
+        data = pd.read_csv(
+            file_name,
+            sep="\t",
+            index_col=0,
+        )
+        X = data.drop("clase", axis=1)
+        self.features = X.columns
+        X = X.to_numpy()
+        y = data["clase"].to_numpy()
+        self.dataset = data
+        self.class_name = "clase"
+        return X, y
+
+
+class DatasetsSurcov:
+    @staticmethod
+    def dataset_names(name):
+        return f"{name}.csv"
+
+    @staticmethod
+    def folder():
+        return "datasets"
+
+    @staticmethod
+    def get_range_features(X, name):
+        return []
+
+    def load(self, name, *args):
+        file_name = os.path.join(self.folder(), self.dataset_names(name))
+        data = pd.read_csv(
+            file_name,
+            index_col=0,
+        )
+        data.dropna(axis=0, how="any", inplace=True)
+        self.columns = data.columns
+        X = data.drop(["class"], axis=1)
+        self.features = X.columns
+        self.class_name = "class"
+        self.dataset = data
+        X = X.to_numpy()
+        y = data["class"].to_numpy()
+        return X, y
+
+
+class Datasets:
+    def __init__(self, dataset_name=None, discretize=None):
+        env_data = EnvData().load()
+        # DatasetsSurcov, DatasetsTanveer, DatasetsArff,...
+        source_name = getattr(
+            __import__(__name__),
+            f"Datasets{env_data['source_data']}",
+        )
+        self.discretize = (
+            env_data["discretize"] == "1"
+            if discretize is None
+            else discretize == "1"
+        )
+        self.dataset = source_name()
+        # initialize self.class_names & self.data_sets
+        class_names, sets = self._init_names(dataset_name)
+        self.class_names = class_names
+        self.data_sets = sets
+        self.states = {}  # states of discretized variables
+
+    def _init_names(self, dataset_name):
+        file_name = os.path.join(self.dataset.folder(), Files.index)
+        self.continuous_features = {}
+        with open(file_name) as f:
+            sets = f.read().splitlines()
+            sets = [x for x in sets if not x.startswith("#")]
+        results = []
+        class_names = []
+        for set_name in sets:
+            try:
+                name, class_name, features = set_name.split(";")
+            except ValueError:
+                class_name = "class"
+                features = "all"
+                name = set_name
+            results.append(name)
+            class_names.append(class_name)
+            features = features.strip()
+            self.continuous_features[name] = features
+        # Set as dataset list the dataset passed as argument
+        if dataset_name is None:
+            return class_names, results
+        try:
+            class_name = class_names[results.index(dataset_name)]
+        except ValueError:
+            raise ValueError(f"Unknown dataset: {dataset_name}")
+        return [class_name], [dataset_name]
+
+    def get_attributes(self, name):
+        tmp = self.discretize
+        self.discretize = False
+        X, y = self.load(name)
+        attr = SimpleNamespace()
+        attr.dataset = name
+        values, counts = np.unique(y, return_counts=True)
+        attr.classes = len(values)
+        attr.samples = X.shape[0]
+        attr.features = X.shape[1]
+        attr.cont_features = len(self.get_continuous_features())
+        attr.distribution = {}
+        comp = ""
+        sep = ""
+        for value, count in zip(values, counts):
+            comp += f"{sep}{count/sum(counts)*100:5.2f}% ({count}) "
+            sep = "/ "
+            attr.distribution[value.item()] = count / sum(counts)
+        attr.balance = comp
+        self.discretize = tmp
+        return attr
+
+    def get_features(self):
+        return self.dataset.features
+
+    def get_states(self, name):
+        return self.states[name] if name in self.states else None
+
+    def get_continuous_features(self):
+        return self.continuous_features_dataset
+
+    def get_class_name(self):
+        return self.dataset.class_name
+
+    def get_dataset(self):
+        return self.dataset.dataset
+
+    def build_states(self, name, X):
+        features = self.get_features()
+        self.states[name] = {
+            features[i]: np.unique(X[:, i]).tolist() for i in range(X.shape[1])
+        }
+
+    def load(self, name, dataframe=False):
+        try:
+            class_name = self.class_names[self.data_sets.index(name)]
+            X, y = self.dataset.load(name, class_name)
+            self.continuous_features_dataset = self.dataset.get_range_features(
+                X, self.continuous_features[name]
+            )
+            if self.discretize:
+                X = self.discretize_dataset(X, y)
+                self.build_states(name, X)
+                dataset = pd.DataFrame(X, columns=self.get_features())
+                dataset[self.get_class_name()] = y
+                self.dataset.dataset = dataset
+            if dataframe:
+                return self.get_dataset()
+            return X, y
+        except (ValueError, FileNotFoundError):
+            raise ValueError(f"Unknown dataset: {name}")
+
+    def discretize_dataset(self, X, y):
+        """Supervised discretization with Fayyad and Irani's MDLP algorithm.
+
+        Parameters
+        ----------
+        X : np.ndarray
+            array (n_samples, n_features) of features
+        y : np.ndarray
+            array (n_samples,) of labels
+
+        Returns
+        -------
+        tuple (X, y) of numpy.ndarray
+        """
+        discretiz = FImdlp()
+        return discretiz.fit_transform(X, y)
+
+    def __iter__(self) -> Diterator:
+        return Diterator(self.data_sets)

benchmark/Experiments.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import json
 import random
 import warnings
@@ -6,7 +7,6 @@ import time
 from datetime import datetime
 from tqdm import tqdm
 import numpy as np
-import pandas as pd
 from sklearn.model_selection import (
     StratifiedKFold,
     KFold,
@@ -14,91 +14,15 @@ from sklearn.model_selection import (
     cross_validate,
 )
 from .Utils import Folders, Files, NO_RESULTS
+from .Datasets import Datasets
 from .Models import Models
 from .Arguments import EnvData
 
 
 class Randomized:
-    seeds = [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]
-
-
-class Diterator:
-    def __init__(self, data):
-        self._stack = data.copy()
-
-    def __next__(self):
-        if len(self._stack) == 0:
-            raise StopIteration()
-        return self._stack.pop(0)
-
-
-class DatasetsTanveer:
     @staticmethod
-    def dataset_names(name):
-        return f"{name}_R.dat"
-
-    @staticmethod
-    def folder():
-        return "data"
-
-    def load(self, name):
-        file_name = os.path.join(self.folder(), self.dataset_names(name))
-        data = pd.read_csv(
-            file_name,
-            sep="\t",
-            index_col=0,
-        )
-        X = data.drop("clase", axis=1).to_numpy()
-        y = data["clase"].to_numpy()
-        return X, y
-
-
-class DatasetsSurcov:
-    @staticmethod
-    def dataset_names(name):
-        return f"{name}.csv"
-
-    @staticmethod
-    def folder():
-        return "datasets"
-
-    def load(self, name):
-        file_name = os.path.join(self.folder(), self.dataset_names(name))
-        data = pd.read_csv(
-            file_name,
-            index_col=0,
-        )
-        data.dropna(axis=0, how="any", inplace=True)
-        self.columns = data.columns
-        col_list = ["class"]
-        X = data.drop(col_list, axis=1).to_numpy()
-        y = data["class"].to_numpy()
-        return X, y
-
-
-class Datasets:
-    def __init__(self, dataset_name=None):
-        envData = EnvData.load()
-        class_name = getattr(
-            __import__(__name__),
-            f"Datasets{envData['source_data']}",
-        )
-        self.dataset = class_name()
-        if dataset_name is None:
-            file_name = os.path.join(self.dataset.folder(), Files.index)
-            with open(file_name) as f:
-                self.data_sets = f.read().splitlines()
-        else:
-            self.data_sets = [dataset_name]
-
-    def load(self, name):
-        try:
-            return self.dataset.load(name)
-        except FileNotFoundError:
-            raise ValueError(f"Unknown dataset: {name}")
-
-    def __iter__(self) -> Diterator:
-        return Diterator(self.data_sets)
+    def seeds():
+        return json.loads(EnvData().load()["seeds"])
 
 
 class BestResults:
@@ -188,8 +112,12 @@ class Experiment:
         platform,
         title,
         progress_bar=True,
+        ignore_nan=True,
+        fit_features=None,
+        discretize=None,
         folds=5,
     ):
+        env_data = EnvData().load()
         today = datetime.now()
         self.time = today.strftime("%H:%M:%S")
         self.date = today.strftime("%Y-%m-%d")
@@ -207,7 +135,18 @@ class Experiment:
         self.score_name = score_name
         self.model_name = model_name
         self.title = title
+        self.ignore_nan = ignore_nan
         self.stratified = stratified == "1"
+        self.discretize = (
+            env_data["discretize"] == "1"
+            if discretize is None
+            else discretize == "1"
+        )
+        self.fit_features = (
+            env_data["fit_features"] == "1"
+            if fit_features is None
+            else fit_features == "1"
+        )
         self.stratified_class = StratifiedKFold if self.stratified else KFold
         self.datasets = datasets
         dictionary = json.loads(hyperparams_dict)
@@ -234,7 +173,7 @@ class Experiment:
         self.platform = platform
         self.progress_bar = progress_bar
         self.folds = folds
-        self.random_seeds = Randomized.seeds
+        self.random_seeds = Randomized.seeds()
         self.results = []
         self.duration = 0
         self._init_experiment()
@@ -242,6 +181,10 @@ class Experiment:
     def get_output_file(self):
         return self.output_file
 
+    @staticmethod
+    def get_python_version():
+        return "{}.{}".format(sys.version_info.major, sys.version_info.minor)
+
     def _build_classifier(self, random_state, hyperparameters):
         self.model = Models.get_model(self.model_name, random_state)
         clf = self.model
@@ -256,7 +199,20 @@ class Experiment:
         self.leaves = []
         self.depths = []
 
-    def _n_fold_crossval(self, X, y, hyperparameters):
+    def _build_fit_params(self, name):
+        if not self.fit_features:
+            return None
+        res = dict(
+            features=self.datasets.get_features(),
+            class_name=self.datasets.get_class_name(),
+        )
+        states = self.datasets.get_states(name)
+        if states is None:
+            return res
+        res["state_names"] = states
+        return res
+
+    def _n_fold_crossval(self, name, X, y, hyperparameters):
         if self.scores != []:
             raise ValueError("Must init experiment before!")
         loop = tqdm(
@@ -273,7 +229,8 @@ class Experiment:
                 shuffle=True, random_state=random_state, n_splits=self.folds
             )
             clf = self._build_classifier(random_state, hyperparameters)
-            self.version = clf.version() if hasattr(clf, "version") else "-"
+            fit_params = self._build_fit_params(name)
+            self.version = Models.get_version(self.model_name, clf)
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore")
                 res = cross_validate(
@@ -281,11 +238,19 @@ class Experiment:
                     X,
                     y,
                     cv=kfold,
+                    fit_params=fit_params,
                     return_estimator=True,
-                    scoring=self.score_name,
+                    scoring=self.score_name.replace("-", "_"),
                 )
-            self.scores.append(res["test_score"])
-            self.times.append(res["fit_time"])
+            if np.isnan(res["test_score"]).any():
+                if not self.ignore_nan:
+                    print(res["test_score"])
+                    raise ValueError("NaN in results")
+                results = res["test_score"][~np.isnan(res["test_score"])]
+            else:
+                results = res["test_score"]
+            self.scores.extend(results)
+            self.times.extend(res["fit_time"])
             for result_item in res["estimator"]:
                 nodes_item, leaves_item, depth_item = Models.get_complexity(
                     self.model_name, result_item
@@ -317,12 +282,15 @@ class Experiment:
         output["model"] = self.model_name
         output["version"] = self.version
         output["stratified"] = self.stratified
+        output["discretized"] = self.discretize
         output["folds"] = self.folds
         output["date"] = self.date
         output["time"] = self.time
         output["duration"] = self.duration
         output["seeds"] = self.random_seeds
         output["platform"] = self.platform
+        output["language_version"] = self.get_python_version()
+        output["language"] = "Python"
         output["results"] = self.results
         with open(self.output_file, "w") as f:
             json.dump(output, f)
@@ -343,7 +311,7 @@ class Experiment:
             n_classes = len(np.unique(y))
             hyperparameters = self.hyperparameters_dict[name][1]
             self._init_experiment()
-            self._n_fold_crossval(X, y, hyperparameters)
+            self._n_fold_crossval(name, X, y, hyperparameters)
             self._add_results(name, hyperparameters, samp, feat, n_classes)
             self._output_results()
         self.duration = time.time() - now
@@ -381,7 +349,7 @@ class GridSearch:
         self.progress_bar = progress_bar
         self.folds = folds
         self.platform = platform
-        self.random_seeds = Randomized.seeds
+        self.random_seeds = Randomized.seeds()
         self.grid_file = os.path.join(
             Folders.results, Files.grid_input(score_name, model_name)
         )

benchmark/Manager.py

@@ -0,0 +1,127 @@
+import os
+from types import SimpleNamespace
+
+import xlsxwriter
+
+from benchmark.Results import Report
+from benchmark.ResultsFiles import Excel
+from benchmark.Utils import Files, Folders, TextColor
+
+
+def get_input(message="", is_test=False):
+    return "test" if is_test else input(message)
+
+
+class Manage:
+    def __init__(self, summary):
+        self.summary = summary
+        self.cmd = SimpleNamespace(
+            quit="q", relist="r", delete="d", hide="h", excel="e"
+        )
+
+    def process_file(self, num, command, path):
+        num = int(num)
+        name = self.summary.data_filtered[num]["file"]
+        file_name_result = os.path.join(path, name)
+        verb1, verb2 = (
+            ("delete", "Deleting")
+            if command == self.cmd.delete
+            else (
+                "hide",
+                "Hiding",
+            )
+        )
+        conf_message = (
+            TextColor.RED
+            + f"Are you sure to {verb1} {file_name_result} (y/n)? "
+        )
+        confirm = get_input(message=conf_message)
+        if confirm == "y":
+            print(TextColor.YELLOW + f"{verb2} {file_name_result}")
+            if command == self.cmd.delete:
+                os.unlink(file_name_result)
+            else:
+                os.rename(
+                    os.path.join(Folders.results, name),
+                    os.path.join(Folders.hidden_results, name),
+                )
+            self.summary.data_filtered.pop(num)
+            get_input(message="Press enter to continue")
+            self.summary.list_results()
+
+    def manage_results(self):
+        """Manage results showed in the summary
+        return True if excel file is created False otherwise
+        """
+
+        message = (
+            TextColor.ENDC
+            + f"Choose option {str(self.cmd).replace('namespace', '')}: "
+        )
+        path = (
+            Folders.hidden_results if self.summary.hidden else Folders.results
+        )
+        book = None
+        max_value = len(self.summary.data_filtered)
+        while True:
+            match get_input(message=message).split():
+                case [self.cmd.relist]:
+                    self.summary.list_results()
+                case [self.cmd.quit]:
+                    if book is not None:
+                        book.close()
+                        return True
+                    return False
+                case [self.cmd.hide, num] if num.isdigit() and int(
+                    num
+                ) < max_value:
+                    if self.summary.hidden:
+                        print("Already hidden")
+                    else:
+                        self.process_file(
+                            num, path=path, command=self.cmd.hide
+                        )
+                case [self.cmd.delete, num] if num.isdigit() and int(
+                    num
+                ) < max_value:
+                    self.process_file(
+                        num=num, path=path, command=self.cmd.delete
+                    )
+                case [self.cmd.excel, num] if num.isdigit() and int(
+                    num
+                ) < max_value:
+                    # Add to excel file result #num
+                    book = self.add_to_excel(num, path, book)
+                case [num] if num.isdigit() and int(num) < max_value:
+                    # Report the result #num
+                    self.report(num, path)
+                case _:
+                    print("Invalid option. Try again!")
+
+    def report(self, num, path):
+        num = int(num)
+        file_name_result = os.path.join(
+            path, self.summary.data_filtered[num]["file"]
+        )
+        try:
+            rep = Report(file_name_result, compare=self.summary.compare)
+            rep.report()
+        except ValueError as e:
+            print(e)
+
+    def add_to_excel(self, num, path, book):
+        num = int(num)
+        file_name_result = os.path.join(
+            path, self.summary.data_filtered[num]["file"]
+        )
+        if book is None:
+            file_name = os.path.join(Folders.excel, Files.be_list_excel)
+            book = xlsxwriter.Workbook(file_name, {"nan_inf_to_errors": True})
+        excel = Excel(
+            file_name=file_name_result,
+            book=book,
+            compare=self.summary.compare,
+        )
+        excel.report()
+        print(f"Added {file_name_result} to {Files.be_list_excel}")
+        return book

benchmark/Models.py

@@ -8,9 +8,31 @@ from sklearn.ensemble import (
 )
 from sklearn.svm import SVC
 from stree import Stree
+
+# from bayesclass.clfs import TAN, KDB, AODE, KDBNew, TANNew, AODENew, BoostAODE
 from wodt import Wodt
 from odte import Odte
 from xgboost import XGBClassifier
+import sklearn
+import xgboost
+
+import random
+
+
+class MockModel(SVC):
+    # Only used for testing
+    def predict(self, X):
+        if random.random() < 0.1:
+            return [float("NaN")] * len(X)
+        return super().predict(X)
+
+    def nodes_leaves(self):
+        return 0, 0
+
+    def fit(self, X, y, **kwargs):
+        kwargs.pop("state_names", None)
+        kwargs.pop("features", None)
+        return super().fit(X, y, **kwargs)
 
 
 class Models:
@@ -18,25 +40,32 @@ class Models:
     def define_models(random_state):
         return {
             "STree": Stree(random_state=random_state),
+            # "TAN": TAN(random_state=random_state),
+            # "KDB": KDB(k=2),
+            # "TANNew": TANNew(random_state=random_state),
+            # "KDBNew": KDBNew(k=2),
+            # "AODENew": AODENew(random_state=random_state),
+            # "AODE": AODE(random_state=random_state),
+            # "BoostAODE": BoostAODE(random_state=random_state),
             "Cart": DecisionTreeClassifier(random_state=random_state),
             "ExtraTree": ExtraTreeClassifier(random_state=random_state),
             "Wodt": Wodt(random_state=random_state),
             "SVC": SVC(random_state=random_state),
             "ODTE": Odte(
-                base_estimator=Stree(random_state=random_state),
+                estimator=Stree(random_state=random_state),
                 random_state=random_state,
             ),
             "BaggingStree": BaggingClassifier(
-                base_estimator=Stree(random_state=random_state),
+                estimator=Stree(random_state=random_state),
                 random_state=random_state,
             ),
             "BaggingWodt": BaggingClassifier(
-                base_estimator=Wodt(random_state=random_state),
+                estimator=Wodt(random_state=random_state),
                 random_state=random_state,
             ),
             "XGBoost": XGBClassifier(random_state=random_state),
             "AdaBoostStree": AdaBoostClassifier(
-                base_estimator=Stree(
+                estimator=Stree(
                     random_state=random_state,
                 ),
                 algorithm="SAMME",
@@ -44,6 +73,7 @@ class Models:
             ),
             "GBC": GradientBoostingClassifier(random_state=random_state),
             "RandomForest": RandomForestClassifier(random_state=random_state),
+            "Mock": MockModel(random_state=random_state),
         }
 
     @staticmethod
@@ -89,3 +119,15 @@ class Models:
             nodes, leaves = result.nodes_leaves()
             depth = result.depth_ if hasattr(result, "depth_") else 0
         return nodes, leaves, depth
+
+    @staticmethod
+    def get_version(name, clf):
+        if hasattr(clf, "version"):
+            return clf.version()
+        if name in ["Cart", "ExtraTree", "RandomForest", "GBC", "SVC"]:
+            return sklearn.__version__
+        elif name.startswith("Bagging") or name.startswith("AdaBoost"):
+            return sklearn.__version__
+        elif name == "XGBoost":
+            return xgboost.__version__
+        return "Error"

benchmark/ResultsBase.py

@@ -0,0 +1,446 @@
+import abc
+import json
+import math
+import os
+from operator import itemgetter
+
+from benchmark.Datasets import Datasets
+from benchmark.Utils import NO_RESULTS, Files, Folders, TextColor
+
+from .Arguments import ALL_METRICS, EnvData
+from .Datasets import Datasets
+from .Experiments import BestResults
+from .Utils import Folders, Symbols
+
+
+class BestResultsEver:
+    def __init__(self):
+        self.data = {}
+        for i in ["Tanveer", "Surcov", "Arff"]:
+            self.data[i] = {}
+            for metric in ALL_METRICS:
+                self.data[i][metric.replace("-", "_")] = ["self", 1.0]
+                self.data[i][metric] = ["self", 1.0]
+        self.data["Tanveer"]["accuracy"] = [
+            "STree_default (liblinear-ovr)",
+            40.282203,
+        ]
+        self.data["Arff"]["accuracy"] = [
+            "STree_default (linear-ovo)",
+            22.109799,
+        ]
+
+    def get_name_value(self, key, score):
+        return self.data[key][score]
+
+
+class BaseReport(abc.ABC):
+    def __init__(self, file_name, best_file=False):
+        self.file_name = file_name
+        if not os.path.isfile(file_name):
+            if not os.path.isfile(os.path.join(Folders.results, file_name)):
+                raise FileNotFoundError(f"{file_name} does not exists!")
+            else:
+                self.file_name = os.path.join(Folders.results, file_name)
+        with open(self.file_name) as f:
+            self.data = json.load(f)
+        self.best_acc_file = best_file
+        if best_file:
+            self.lines = self.data
+        else:
+            self.lines = self.data["results"]
+            self.score_name = self.data["score_name"]
+        self.__load_env_data()
+        self.__compute_best_results_ever()
+        self._compare_totals = {}
+
+    def __load_env_data(self):
+        # Set the labels for nodes, leaves, depth
+        env_data = EnvData().load()
+        self.nodes_label = env_data["nodes"]
+        self.leaves_label = env_data["leaves"]
+        self.depth_label = env_data["depth"]
+        self.key = env_data["source_data"]
+        self.margin = float(env_data["margin"])
+
+    def __compute_best_results_ever(self):
+        best = BestResultsEver()
+        self.best_score_name, self.best_score_value = best.get_name_value(
+            self.key, self.score_name
+        )
+
+    def _get_accuracy(self, item):
+        return self.data[item][0] if self.best_acc_file else item["score"]
+
+    def report(self):
+        self.header()
+        accuracy_total = 0.0
+        for result in self.lines:
+            self.print_line(result)
+            accuracy_total += self._get_accuracy(result)
+        self.footer(accuracy_total)
+
+    def _load_best_results(self, score, model):
+        best = BestResults(score, model, Datasets())
+        self.best_results = best.load({})
+
+    def _compute_status(self, dataset, accuracy: float):
+        status = " "
+        if self.compare:
+            # Compare with best results
+            best = self.best_results[dataset][0]
+            if accuracy == best:
+                status = Symbols.equal_best
+            elif accuracy > best:
+                status = Symbols.better_best
+        else:
+            # compare with dataset label distribution only if its a binary one
+            # down_arrow if accuracy is less than the ZeroR
+            # black_star if accuracy is greater than the ZeroR + margin%
+            if self.score_name == "accuracy":
+                dt = Datasets()
+                attr = dt.get_attributes(dataset)
+                if attr.classes == 2:
+                    max_category = max(attr.distribution.values())
+                    max_value = max_category * (1 + self.margin)
+                    if max_value > 1:
+                        max_value = 0.9995
+                    status = (
+                        Symbols.cross
+                        if accuracy <= max_value
+                        else (
+                            Symbols.upward_arrow
+                            if accuracy > max_value
+                            else " "
+                        )
+                    )
+        if status != " ":
+            if status not in self._compare_totals:
+                self._compare_totals[status] = 1
+            else:
+                self._compare_totals[status] += 1
+        return status
+
+    def _status_meaning(self, status):
+        meaning = {
+            Symbols.equal_best: "Equal to best",
+            Symbols.better_best: "Better than best",
+            Symbols.cross: "Less than or equal to ZeroR",
+            Symbols.upward_arrow: f"Better than ZeroR + "
+            f"{self.margin*100:3.1f}%",
+        }
+        return meaning[status]
+
+    def _get_best_accuracy(self):
+        return self.best_score_value
+
+    def _get_message_best_accuracy(self):
+        return f"{self.score_name} compared to {self.best_score_name} .:"
+
+    @abc.abstractmethod
+    def header(self) -> None:
+        pass
+
+    @abc.abstractmethod
+    def print_line(self, result) -> None:
+        pass
+
+    @abc.abstractmethod
+    def footer(self, accuracy: float) -> None:
+        pass
+
+
+class StubReport(BaseReport):
+    def __init__(self, file_name, compare=False):
+        self.compare = compare
+        super().__init__(file_name=file_name, best_file=False)
+        if self.compare:
+            self._load_best_results(self.score_name, self.data["model"])
+
+    def print_line(self, line) -> None:
+        pass
+
+    def header(self) -> None:
+        self.title = self.data["title"]
+        self.duration = self.data["duration"]
+        self.model = self.data["model"]
+        self.date = self.data["date"]
+        self.time = self.data["time"]
+        self.metric = self.data["score_name"]
+        self.platform = self.data["platform"]
+
+    def footer(self, accuracy: float) -> None:
+        self.accuracy = accuracy
+        self.score = accuracy / self._get_best_accuracy()
+
+
+class Summary:
+    def __init__(self, hidden=False, compare=False) -> None:
+        self.results = Files.get_all_results(hidden=hidden)
+        self.data = []
+        self.data_filtered = []
+        self.datasets = {}
+        self.models = set()
+        self.hidden = hidden
+        self.compare = compare
+
+    def get_models(self):
+        return sorted(self.models)
+
+    def acquire(self, given_score="any") -> None:
+        """Get all results"""
+        for result in self.results:
+            (
+                score,
+                model,
+                platform,
+                date,
+                time,
+                stratified,
+            ) = Files().split_file_name(result)
+            if given_score in ("any", score):
+                self.models.add(model)
+                report = StubReport(
+                    os.path.join(
+                        (
+                            Folders.hidden_results
+                            if self.hidden
+                            else Folders.results
+                        ),
+                        result,
+                    )
+                )
+                report.report()
+                entry = dict(
+                    score=score,
+                    model=model,
+                    title=report.title,
+                    platform=platform,
+                    date=date,
+                    time=time,
+                    stratified=stratified,
+                    file=result,
+                    metric=report.score,
+                    duration=report.duration,
+                )
+                self.datasets[result] = report.lines
+                self.data.append(entry)
+
+    def get_results_criteria(
+        self, score, model, input_data, sort_key, number, nan=False
+    ):
+        data = self.data.copy() if input_data is None else input_data
+        if score:
+            data = [x for x in data if x["score"] == score]
+        if model:
+            data = [x for x in data if x["model"] == model]
+        if nan:
+            data = [x for x in data if x["metric"] != x["metric"]]
+        keys = (
+            itemgetter(sort_key, "time")
+            if sort_key == "date"
+            else itemgetter(sort_key, "date", "time")
+        )
+        data = sorted(data, key=keys, reverse=True)
+        if number > 0:
+            data = data[:number]
+        return data
+
+    def list_results(
+        self,
+        score=None,
+        model=None,
+        input_data=None,
+        sort_key="date",
+        number=0,
+        nan=False,
+    ) -> None:
+        """Print the list of results"""
+        if self.data_filtered == []:
+            self.data_filtered = self.get_results_criteria(
+                score, model, input_data, sort_key, number, nan=nan
+            )
+        if self.data_filtered == []:
+            raise ValueError(NO_RESULTS)
+        max_file = max(len(x["file"]) for x in self.data_filtered)
+        max_title = max(len(x["title"]) for x in self.data_filtered)
+        if self.hidden:
+            color1 = TextColor.GREEN
+            color2 = TextColor.YELLOW
+        else:
+            color1 = TextColor.LINE1
+            color2 = TextColor.LINE2
+        print(color1, end="")
+        print(
+            f" #  {'Date':10s} {'File':{max_file}s} {'Score':8s} "
+            f"{'Time(h)':7s} {'Title':s}"
+        )
+        print(
+            "===",
+            "=" * 10
+            + " "
+            + "=" * max_file
+            + " "
+            + "=" * 8
+            + " "
+            + "=" * 7
+            + " "
+            + "=" * max_title,
+        )
+        print(
+            "\n".join(
+                [
+                    (color2 if n % 2 == 0 else color1) + f"{n:3d} "
+                    f"{x['date']} {x['file']:{max_file}s} "
+                    f"{x['metric']:8.5f} "
+                    f"{x['duration']/3600:7.3f} "
+                    f"{x['title']}"
+                    for n, x in enumerate(self.data_filtered)
+                ]
+            )
+        )
+
+    def show_result(self, data: dict, title: str = "") -> None:
+        def whites(n: int) -> str:
+            return " " * n + color1 + "*"
+
+        if data == {}:
+            print(f"** {title} has No data **")
+            return
+        color1 = TextColor.CYAN
+        color2 = TextColor.YELLOW
+        file_name = data["file"]
+        metric = data["metric"]
+        result = StubReport(os.path.join(Folders.results, file_name))
+        length = 81
+        print(color1 + "*" * length)
+        if title != "":
+            print(
+                "*"
+                + color2
+                + TextColor.BOLD
+                + f"{title:^{length - 2}s}"
+                + TextColor.ENDC
+                + color1
+                + "*"
+            )
+            print("*" + "-" * (length - 2) + "*")
+        print("*" + whites(length - 2))
+        print(
+            "* "
+            + color2
+            + f"{result.data['title']:^{length - 4}}"
+            + color1
+            + " *"
+        )
+        print("*" + whites(length - 2))
+        print(
+            "* Model: "
+            + color2
+            + f"{result.data['model']:15s} "
+            + color1
+            + "Ver. "
+            + color2
+            + f"{result.data['version']:10s} "
+            + color1
+            + "Score: "
+            + color2
+            + f"{result.data['score_name']:10s} "
+            + color1
+            + "Metric: "
+            + color2
+            + f"{metric:10.7f}"
+            + whites(length - 78)
+        )
+        print(color1 + "*" + whites(length - 2))
+        print(
+            "* Date : "
+            + color2
+            + f"{result.data['date']:15s}"
+            + color1
+            + " Time: "
+            + color2
+            + f"{result.data['time']:18s} "
+            + color1
+            + "Time Spent: "
+            + color2
+            + f"{result.data['duration']:9,.2f}"
+            + color1
+            + " secs."
+            + whites(length - 78)
+        )
+        seeds = str(result.data["seeds"])
+        seeds_len = len(seeds)
+        print(
+            "* Seeds: "
+            + color2
+            + f"{seeds:{seeds_len}s} "
+            + color1
+            + "Platform: "
+            + color2
+            + f"{result.data['platform']:17s} "
+            + whites(length - 79)
+        )
+        print(
+            "* Stratified: "
+            + color2
+            + f"{str(result.data['stratified']):15s}"
+            + whites(length - 30)
+        )
+        print("* " + color2 + f"{file_name:60s}" + whites(length - 63))
+        print(color1 + "*" + whites(length - 2))
+        print(color1 + "*" * length)
+
+    def best_results(self, criterion=None, value=None, score="accuracy", n=10):
+        # First filter the same score results (accuracy, f1, ...)
+        haystack = [x for x in self.data if x["score"] == score]
+        haystack = (
+            haystack
+            if criterion is None or value is None
+            else [x for x in haystack if x[criterion] == value]
+        )
+        if haystack == []:
+            raise ValueError(NO_RESULTS)
+        return (
+            sorted(
+                haystack,
+                key=lambda x: -1.0 if math.isnan(x["metric"]) else x["metric"],
+                reverse=True,
+            )[:n]
+            if len(haystack) > 0
+            else {}
+        )
+
+    def best_result(
+        self, criterion=None, value=None, score="accuracy"
+    ) -> dict:
+        return self.best_results(criterion, value, score)[0]
+
+    def best_results_datasets(self, score="accuracy") -> dict:
+        """Get the best results for each dataset"""
+        dt = Datasets()
+        best_results = {}
+        for dataset in dt:
+            best_results[dataset] = (1, "", "", "")
+        haystack = [x for x in self.data if x["score"] == score]
+        # Search for the best results for each dataset
+        for entry in haystack:
+            for dataset in self.datasets[entry["file"]]:
+                if dataset["score"] < best_results[dataset["dataset"]][0]:
+                    best_results[dataset["dataset"]] = (
+                        dataset["score"],
+                        dataset["hyperparameters"],
+                        entry["file"],
+                        entry["title"],
+                    )
+        return best_results
+
+    def show_top(self, score="accuracy", n=10):
+        try:
+            self.list_results(
+                score=score,
+                input_data=self.best_results(score=score, n=n),
+                sort_key="metric",
+            )
+        except ValueError as e:
+            print(e)

benchmark/Utils.py

@@ -1,7 +1,8 @@
 import os
+import sys
 import subprocess
 
-BEST_ACCURACY_STREE = 40.282203
+PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor)
 NO_RESULTS = "** No results found **"
 NO_ENV = "File .env not found"
 
@@ -12,6 +13,9 @@ class Folders:
     exreport = "exreport"
     report = os.path.join(exreport, "exreport_output")
     img = "img"
+    excel = "excel"
+    sql = "sql"
+    current = os.getcwd()
 
     @staticmethod
     def src():
@@ -26,6 +30,8 @@ class Files:
     exreport_pdf = "Rplots.pdf"
     benchmark_r = "benchmark.r"
     dot_env = ".env"
+    datasets_report_excel = "ReportDatasets.xlsx"
+    be_list_excel = "some_results.xlsx"
 
     @staticmethod
     def exreport_output(score):
@@ -103,7 +109,8 @@ class Files:
             )
         return None
 
-    def get_all_results(self, hidden) -> list[str]:
+    @staticmethod
+    def get_all_results(hidden) -> list[str]:
         result_path = os.path.join(
             ".", Folders.hidden_results if hidden else Folders.results
         )
@@ -112,7 +119,7 @@ class Files:
         else:
             raise ValueError(f"{result_path} does not exist")
         result = []
-        prefix, suffix = self.results_suffixes()
+        prefix, suffix = Files.results_suffixes()
         for result_file in files_list:
             if result_file.startswith(prefix) and result_file.endswith(suffix):
                 result.append(result_file)
@@ -123,6 +130,9 @@ class Symbols:
     check_mark = "\N{heavy check mark}"
     exclamation = "\N{heavy exclamation mark symbol}"
     black_star = "\N{black star}"
+    cross = "\N{Ballot X}"
+    upward_arrow = "\N{Black-feathered north east arrow}"
+    down_arrow = "\N{downwards black arrow}"
     equal_best = check_mark
     better_best = black_star
 
@@ -143,3 +153,7 @@ class TextColor:
     ENDC = "\033[0m"
     BOLD = "\033[1m"
     UNDERLINE = "\033[4m"
+    WHITE = "\033[97m"
+    GREY = "\033[90m"
+    BLACK = "\033[90m"
+    DEFAULT = "\033[99m"

benchmark/__init__.py

@@ -1,9 +1,17 @@
-from .Experiments import Experiment, Datasets, DatasetsSurcov, DatasetsTanveer
-from .Results import Report, Summary
+from .ResultsBase import Summary
+from .Datasets import (
+    Datasets,
+    DatasetsSurcov,
+    DatasetsTanveer,
+    DatasetsArff,
+)
+from .Experiments import Experiment
+from .Results import Report
+from ._version import __version__
 
 __author__ = "Ricardo Montañana Gómez"
-__copyright__ = "Copyright 2020-2022, Ricardo Montañana Gómez"
+__copyright__ = "Copyright 2020-2024, Ricardo Montañana Gómez"
 __license__ = "MIT License"
 __author_email__ = "ricardo.montanana@alu.uclm.es"
 
-__all__ = ["Experiment", "Datasets", "Report", "Summary"]
+__all__ = ["Experiment", "Datasets", "Report", "Summary", __version__]

benchmark/_version.py

@@ -1 +1 @@
-__version__ = "0.1.1"
+__version__ = "1.0.1"

benchmark/scripts/app/app.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+from benchmark.Arguments import EnvData
+from flask import Flask
+from .main import main, OUTPUT
+
+FRAMEWORK = "framework"
+FRAMEWORKS = "frameworks"
+TEST = "test"
+
+
+def create_app(output="local"):
+    app = Flask(__name__)
+    config = EnvData().load()
+    app.register_blueprint(main)
+    app.config[FRAMEWORK] = config[FRAMEWORK]
+    app.config[FRAMEWORKS] = ["bootstrap", "bulma"]
+    app.config[OUTPUT] = output
+    app.jinja_env.auto_reload = True
+    app.config["TEMPLATES_AUTO_RELOAD"] = True
+    return app

benchmark/scripts/app/main.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+import os
+import json
+import shutil
+import xlsxwriter
+from dotenv import dotenv_values
+from benchmark.Utils import Files, Folders
+from benchmark.Arguments import EnvData
+from benchmark.ResultsBase import StubReport
+from benchmark.ResultsFiles import Excel, ReportDatasets
+from benchmark.Datasets import Datasets
+from flask import Blueprint, current_app, send_file
+from flask import render_template, request, redirect, url_for
+
+
+main = Blueprint("main", __name__)
+FRAMEWORK = "framework"
+FRAMEWORKS = "frameworks"
+OUTPUT = "output"
+TEST = "test"
+
+
+class AjaxResponse:
+    def __init__(self, success, file_name, code=200):
+        self.success = success
+        self.file_name = file_name
+        self.code = code
+
+    def to_string(self):
+        return (
+            json.dumps(
+                {
+                    "success": self.success,
+                    "file": self.file_name,
+                    "output": current_app.config[OUTPUT],
+                }
+            ),
+            self.code,
+            {"ContentType": "application/json"},
+        )
+
+
+def process_data(file_name, compare, data):
+    report = StubReport(
+        os.path.join(Folders.results, file_name), compare=compare
+    )
+    new_list = []
+    for result in data["results"]:
+        symbol = report._compute_status(result["dataset"], result["score"])
+        result["symbol"] = symbol if symbol != " " else " "
+        new_list.append(result)
+    data["results"] = new_list
+    # Compute summary with explanation of symbols
+    summary = {}
+    for key, value in report._compare_totals.items():
+        summary[key] = (report._status_meaning(key), value)
+    return summary
+
+
+@main.route("/index/<compare>")
+@main.route("/")
+def index(compare="False"):
+    # Get a list of files in a directory
+    files = {}
+    names = Files.get_all_results(hidden=False)
+    for name in names:
+        report = StubReport(os.path.join(Folders.results, name))
+        report.report()
+        files[name] = {
+            "duration": report.duration,
+            "score": report.score,
+            "title": report.title,
+        }
+    candidate = current_app.config[FRAMEWORKS].copy()
+    candidate.remove(current_app.config[FRAMEWORK])
+    return render_template(
+        "select.html",
+        files=files,
+        candidate=candidate[0],
+        framework=current_app.config[FRAMEWORK],
+        compare=compare.capitalize() == "True",
+    )
+
+
+@main.route("/datasets/<compare>")
+def datasets(compare):
+    dt = Datasets()
+    datos = []
+    for dataset in dt:
+        datos.append(dt.get_attributes(dataset))
+    return render_template(
+        "datasets.html",
+        datasets=datos,
+        compare=compare,
+        framework=current_app.config[FRAMEWORK],
+    )
+
+
+@main.route("/showfile/<file_name>/<compare>")
+def showfile(file_name, compare, back=None):
+    compare = compare.capitalize() == "True"
+    back = request.args["url"] if back is None else back
+    app_config = dotenv_values(".env")
+    with open(os.path.join(Folders.results, file_name)) as f:
+        data = json.load(f)
+    try:
+        summary = process_data(file_name, compare, data)
+    except Exception as e:
+        return render_template("error.html", message=str(e), compare=compare)
+    return render_template(
+        "report.html",
+        data=data,
+        file=file_name,
+        summary=summary,
+        framework=current_app.config[FRAMEWORK],
+        back=back,
+        app_config=app_config,
+    )
+
+
+@main.route("/show", methods=["post"])
+def show():
+    selected_file = request.form["selected-file"]
+    compare = request.form["compare"]
+    return showfile(
+        file_name=selected_file,
+        compare=compare,
+        back=url_for(
+            "main.index", compare=compare, output=current_app.config[OUTPUT]
+        ),
+    )
+
+
+@main.route("/excel", methods=["post"])
+def excel():
+    selected_files = request.json["selectedFiles"]
+    compare = request.json["compare"]
+    book = None
+    if selected_files[0] == "datasets":
+        # Create a list of datasets
+        report = ReportDatasets(excel=True, output=False)
+        report.report()
+        excel_name = os.path.join(Folders.excel, Files.datasets_report_excel)
+        if current_app.config[OUTPUT] == "local":
+            Files.open(excel_name, test=current_app.config[TEST])
+        return AjaxResponse(True, Files.datasets_report_excel).to_string()
+    try:
+        for file_name in selected_files:
+            file_name_result = os.path.join(Folders.results, file_name)
+            if book is None:
+                file_excel = os.path.join(Folders.excel, Files.be_list_excel)
+                book = xlsxwriter.Workbook(
+                    file_excel, {"nan_inf_to_errors": True}
+                )
+            excel = Excel(
+                file_name=file_name_result,
+                book=book,
+                compare=compare,
+            )
+            excel.report()
+    except Exception as e:
+        if book is not None:
+            book.close()
+        return AjaxResponse(
+            False, "Could not create excel file, " + str(e)
+        ).to_string()
+    if book is not None:
+        book.close()
+    if current_app.config[OUTPUT] == "local":
+        Files.open(file_excel, test=current_app.config[TEST])
+    return AjaxResponse(True, Files.be_list_excel).to_string()
+
+
+@main.route("/download/<file_name>")
+def download(file_name):
+    src = os.path.join(Folders.current, Folders.excel, file_name)
+    dest = os.path.join(
+        Folders.src(), "scripts", "app", "static", "excel", file_name
+    )
+    shutil.copyfile(src, dest)
+    return send_file(dest, as_attachment=True)
+
+
+@main.route("/config/<framework>/<compare>")
+def config(framework, compare):
+    if framework not in current_app.config[FRAMEWORKS]:
+        message = f"framework {framework} not supported"
+        return render_template("error.html", message=message)
+    env = EnvData()
+    env.load()
+    env.args[FRAMEWORK] = framework
+    env.save()
+    current_app.config[FRAMEWORK] = framework
+    return redirect(url_for("main.index", compare=compare))
+
+
+@main.route("/best_results/<file>/<compare>")
+def best_results(file, compare):
+    compare = compare.capitalize() == "True"
+    try:
+        with open(os.path.join(Folders.results, file)) as f:
+            data = json.load(f)
+    except Exception as e:
+        return render_template("error.html", message=str(e), compare=compare)
+    return render_template(
+        "report_best.html",
+        data=data,
+        compare=compare,
+        framework=current_app.config[FRAMEWORK],
+    )

benchmark/scripts/app/static/css/main.css

@@ -0,0 +1,30 @@
+.alternate-font {
+  font-family: Arial;
+}
+
+tbody {
+  font-family: Courier;
+}
+
+.tag {
+  cursor: pointer;
+}
+
+.ajaxLoading {
+  cursor: progress !important;
+}
+
+#file-table tbody tr.selected td {
+  background-color: #0dcaf0;
+  color: white;
+}
+
+#report-table tbody tr.selected td {
+  background-color: #0dcaf0;
+  color: white;
+}
+
+.btn-small {
+  padding: 0.25rem 0.5rem;
+  font-size: 0.75rem;
+}

benchmark/scripts/app/static/excel/.gitignore

@@ -0,0 +1 @@
+*.xlsx

benchmark/scripts/app/static/js/excelFiles.js

@@ -0,0 +1,29 @@
+function excelFiles(selectedFiles, compare) {
+    var data = {
+        "selectedFiles": selectedFiles,
+        "compare": compare
+    };
+    // send data to server with ajax post
+    $.ajax({
+        type:'POST',
+        url:'/excel',
+        data: JSON.stringify(data),
+        contentType: "application/json",
+        dataType: 'json',
+        success: function(data){
+            if (data.success) {
+                if (data.output == "local") {
+                    alert("Se ha generado el archivo " + data.file);
+                } else {
+                    window.open('/download/' + data.file, "_blank");
+                }
+            } else {
+                alert(data.file);
+            }
+        },
+        error: function (xhr, ajaxOptions, thrownError) {
+            var mensaje = JSON.parse(xhr.responseText || '{\"mensaje\": \"Error indeterminado\"}');
+            alert(mensaje.mensaje);
+        }
+    });
+}

benchmark/scripts/app/templates/base_bootstrap.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>{{ title }}</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-KK94CHFLLe+nY2dmCWGMq91rCGa5gtU4mk92HdvYe+M/SXH301p5ILy+dN9+nJOZ" crossorigin="anonymous" />
+    <link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,500,600" rel="stylesheet" />
+    <link rel="stylesheet" href="https://cdn.datatables.net/1.10.25/css/jquery.dataTables.min.css" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/MaterialDesign-Webfont/7.1.96/css/materialdesignicons.css" integrity="sha512-lD1LHcZ8tFHvMFNeo6qOLY/HjzSPCasPJOAoir22byDxlZI1R71S5lZel8zRL2TZ+Dut1wOHfYgSU2lHXuL00w==" crossorigin="anonymous" referrerpolicy="no-referrer" />
+     <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}" />
+  </head>
+  <body>
+    {% block content %}
+
+    {% endblock %}
+  </body>
+  <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+  {% block jscript %}
+
+  {% endblock %}
+</html>

benchmark/scripts/app/templates/base_bulma.html

@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>{{ title }}</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.3/css/bulma.min.css" />
+    <link rel="stylesheet" href="https://cdn.datatables.net/1.10.25/css/jquery.dataTables.min.css" />
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/MaterialDesign-Webfont/7.1.96/css/materialdesignicons.css" integrity="sha512-lD1LHcZ8tFHvMFNeo6qOLY/HjzSPCasPJOAoir22byDxlZI1R71S5lZel8zRL2TZ+Dut1wOHfYgSU2lHXuL00w==" crossorigin="anonymous" referrerpolicy="no-referrer" />
+    <link rel="stylesheet" href="{{ url_for('static', filename='css/main.css') }}" />
+  </head>
+  <body>
+    {% block content %}
+
+    {% endblock %}
+  </body>
+  <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
+  {% block jscript %}
+
+  {% endblock %}
+</html>

benchmark/scripts/app/templates/datasets.html

@@ -0,0 +1,68 @@
+{% extends 'base_' ~ framework ~ '.html' %}
+{% macro javascript(file) %}
+  <script src="{{ url_for('static', filename=file) }}"></script>
+{% endmacro %}
+{% if framework == 'bootstrap' %}
+  {% set button_class = 'btn btn-primary btn-small' %}
+  {% set h1_class = 'text-center' %}
+  {% set table_class = 'table table-striped table-hover table-bordered' %}
+  {% set head_class = 'bg-primary text-white' %}
+  {% set text_right = 'text-end' %}
+  {% set container = 'container' %}
+  {% set selected = 'selected' %}
+  {%- macro header(title, close, url) -%}
+    <div class="p-4 bg-primary text-white">
+      {%- if close -%}
+        <button type="button" class="btn-close" aria-label="Close" onclick="location.href = '{{ url }}'"></button>
+      {%- endif -%}
+      <h1 class="alternate-font">{{ title }}</h1>
+    </div>
+  {%- endmacro -%}
+{% else %}
+  {% set button_class = 'button is-primary is-small' %}
+  {% set h1_class = 'title is-1 has-text-centered' %}
+  {% set table_class = 'table is-striped is-hoverable cell-border is-bordered' %}
+  {% set head_class = 'is-selected' %}
+  {% set text_right = 'has-text-right' %}
+  {% set container = 'container' %}
+  {% set selected = 'is-selected' %}
+  {%- macro header(title, close, url) -%}
+    <div class="hero is-info is-bold">
+      <div class="hero-body">
+        {%- if close -%}
+          <button class="delete is-large" onclick="location.href = '{{ url }}'"></button>
+        {%- endif -%}
+        <h1 class="is-size-3 alternate-font">{{ title }}</h1>
+      </div>
+    </div>
+  {%- endmacro -%}
+{% endif %}
+{% block content %}
+  <div class="{{ container }}">
+    {{ header('Benchmark Datasets Report', True, url_for('main.index', compare = compare)) }}
+    <button class="{{ button_class }}" onclick="excelFiles(['datasets'], false)"><i class="mdi mdi-file-excel"></i> Excel</button>
+    {% include 'partials/datasets_table.html' %}
+  </div>
+{% endblock %}
+{% block jscript %}
+{{ javascript("js/excelFiles.js") }}
+  <script>
+    $(document).ready(function () {
+      $(document).ajaxStart(function(){ 
+          $("body").addClass('ajaxLoading');
+      });
+      $(document).ajaxStop(function(){ 
+          $("body").removeClass('ajaxLoading');
+      });
+    });
+    // Check if row is selected
+    $('#file-table tbody').on('click', 'tr', function () {
+      if ($(this).hasClass('{{ selected }}')) {
+          $(this).removeClass('{{ selected }}');
+      } else {
+          $('#file-table tbody tr.{{ selected }}').removeClass("{{ selected }}")
+          $(this).addClass('{{ selected }}');
+      }
+    });
+  </script>
+{% endblock %}

benchmark/scripts/app/templates/error.html

@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Error</title>
+        <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha3/dist/css/bootstrap.min.css" rel="stylesheet"
+            integrity="sha384-KK94CHFLLe+nY2dmCWGMq91rCGa5gtU4mk92HdvYe+M/SXH301p5ILy+dN9+nJOZ" crossorigin="anonymous">
+    </head>
+    <body>
+        <div class="container">
+            <div class="alert alert-danger my-5" role="alert">
+
+                <h4 class="alert-heading"><button class="btn-close btn-sm" type="button"
+                        onclick="location.href='/index/{{ compare }}';"></button>Error</h4>
+                <p>There was an error processing action, {{ message }}. Please try again later.</p>
+                <hr>
+                <p class="mb-0">If the problem persists, please contact support.</p>
+            </div>
+        </div>
+    </body>
+</html>

benchmark/scripts/app/templates/js/report.js

@@ -0,0 +1,22 @@
+$(document).ready(function () {
+    // Check if row is selected
+    $('#report-table tbody').on('click', 'tr', function () {
+        if ($(this).hasClass('{{ selected }}')) {
+            $(this).removeClass('{{ selected }}');
+        } else {
+            $('#report-table tbody tr.{{ selected }}').removeClass("{{ selected }}")
+            $(this).addClass('{{ selected }}');
+        }
+    });
+    $(document).ajaxStart(function(){ 
+        $("body").addClass('ajaxLoading');
+    });
+    $(document).ajaxStop(function(){ 
+        $("body").removeClass('ajaxLoading');
+    });
+  });
+  function excelFile() {
+    var selectedFiles = ["{{ file }}"];
+    var compare = "{{ compare }}" == "True";
+    excelFiles(selectedFiles, compare)
+}

benchmark/scripts/app/templates/js/select.js

@@ -0,0 +1,97 @@
+$(document).ready(function () {
+  var table = $("#file-table").DataTable({
+    paging: true,
+    searching: true,
+    ordering: true,
+    info: true,
+    "select.items": "row",
+    pageLength: 25,
+    columnDefs: [
+      {
+        targets: 8,
+        orderable: false,
+      },
+    ],
+    //"language": {
+    //    "lengthMenu": "_MENU_"
+    //}
+  });
+  $('#file-table').on( 'draw.dt', function () {
+    enable_disable_best_buttons();
+  } );
+  // Check if row is selected
+  $("#file-table tbody").on("click", "tr", function () {
+    if ($(this).hasClass("{{ select.selected() }}")) {
+      $(this).removeClass("{{ select.selected() }}");
+    } else {
+      table
+        .$("tr.{{ select.selected() }}")
+        .removeClass("{{ select.selected() }}");
+      $(this).addClass("{{ select.selected() }}");
+    }
+  });
+  // Show file with doubleclick
+  $("#file-table tbody").on("dblclick", "tr", function () {
+    showFile($(this).attr("id"));
+  });
+  $(document).ajaxStart(function () {
+    $("body").addClass("ajaxLoading");
+  });
+  $(document).ajaxStop(function () {
+    $("body").removeClass("ajaxLoading");
+  });
+  $('#compare').change(function() {
+    enable_disable_best_buttons();
+  });
+  enable_disable_best_buttons();
+});
+function enable_disable_best_buttons(){
+  if ($('#compare').is(':checked')) {
+    $("[name='best_buttons']").addClass("tag is-link is-normal");
+      $("[name='best_buttons']").removeAttr("hidden");
+    } else {
+      $("[name='best_buttons']").removeClass("tag is-link is-normal");
+      $("[name='best_buttons']").attr("hidden", true);
+    }
+}
+function showFile(selectedFile) {
+  var form = $(
+    '<form action="/show" method="post">' +
+      '<input type="hidden" name="selected-file" value="' +
+      selectedFile +
+      '" />' +
+      '<input type="hidden" name="compare" value=' +
+      $("#compare").is(":checked") +
+      " />" +
+      "</form>"
+  );
+  $("body").append(form);
+  form.submit();
+}
+function excel() {
+  var checkbox = document.getElementsByName("selected_files");
+  var selectedFiles = [];
+  for (var i = 0; i < checkbox.length; i++) {
+    if (checkbox[i].checked) {
+      selectedFiles.push(checkbox[i].value);
+    }
+  }
+  if (selectedFiles.length == 0) {
+    alert("Select at least one file");
+    return;
+  }
+  var compare = $("#compare").is(":checked");
+  excelFiles(selectedFiles, compare);
+}
+function setCheckBoxes(value) {
+  var checkbox = document.getElementsByName("selected_files");
+  for (i = 0; i < checkbox.length; i++) {
+    checkbox[i].checked = value;
+  }
+}
+function redirectDouble(route, parameter) {
+  location.href = "/"+ route + "/" + parameter + "/" + $("#compare").is(":checked");
+}
+function redirectSimple(route) {
+  location.href = "/" + route + "/" + $("#compare").is(":checked");
+}

benchmark/scripts/app/templates/partials/cfg_select_bootstrap.jinja

@@ -0,0 +1,56 @@
+{%- macro header(title, close=False, url="") -%}
+<div class="p-4 bg-primary text-white">
+{%- if close -%}
+    <button type="button" class="btn-close" aria-label="Close" onclick="location.href = '{{url}}'"></button>
+{%- endif -%}
+<h1 class="alternate-font">{{ title }}</h1>
+</div>
+{%- endmacro -%}
+{%- macro get_table_class() -%}
+table table-striped table-hover table-bordered
+{%- endmacro -%}
+{%- macro icon(icon_name) -%}
+<i class="mdi mdi-{{icon_name}}"></i>
+{%- endmacro -%}
+{%- macro get_button(text, action) -%}
+    <button class="btn btn-primary btn-small" onclick="{{ action }}">{{ text|safe }}</button>
+{%- endmacro -%}
+{%- macro get_button_class() -%}
+    button btn-primary btn-small
+{%- endmacro %}
+{%- macro get_button_tag(icon_name, method, visible=True, name="") -%}
+    <button class="btn btn-primary btn-small" onclick="{{ method }}" {{ "" if visible else "hidden='true'" }} {{ "" if name=="" else "name='" + name +"'"}}><i class="mdi mdi-{{ icon_name }}"></i></button>
+{%- endmacro -%}
+{%- macro get_button_reset() -%}
+<button class="btn btn-primary btn-small btn-danger" onclick="setCheckBoxes(false)"><i class="mdi mdi-checkbox-multiple-blank"></i></button>
+{%- endmacro -%}
+{%- macro get_button_all() -%}
+<button class="btn btn-primary btn-small btn-success" onclick="setCheckBoxes(true)"><i class="mdi mdi-checkbox-multiple-marked"></i></button>
+{%- endmacro -%}
+{%- macro get_tag_class() -%}
+badge bg-info bg-small
+{%- endmacro -%}
+{%- macro get_container_class() -%}
+container-fluid
+{%- endmacro -%}
+{%- macro selected() -%}
+selected
+{%- endmacro -%}
+{%- macro get_level_class() -%}
+navbar
+{%- endmacro -%}
+{%- macro get_align_right() -%}
+text-end
+{%- endmacro -%}
+{%- macro get_left_position() -%}
+float-left
+{%- endmacro -%}
+{%- macro get_right_position() -%}
+float-right
+{%- endmacro -%}
+{%- macro get_row_head_class() -%}
+bg-primary text-white
+{%- endmacro -%}
+{%- macro get_align_center() -%}
+text-center
+{%- endmacro -%}

benchmark/scripts/app/templates/partials/cfg_select_bulma.jinja

@@ -0,0 +1,58 @@
+{%- macro header(title, close=False, url="") -%}
+<div class="hero is-info is-bold">
+<div class="hero-body">
+    {%- if close -%}
+    <button class="delete is-large" onclick="location.href = '{{ url }}'"></button>
+    {%- endif -%}
+    <h1 class="is-size-3 alternate-font">{{ title }}</h1>
+</div>
+</div>
+{%- endmacro -%}
+{%- macro get_table_class() -%}
+table is-striped is-hoverable cell-border is-bordered
+{%- endmacro -%}
+{%- macro icon(icon_name) -%}
+<i class="mdi mdi-{{icon_name}}"></i>
+{%- endmacro -%}
+{%- macro get_button(text, action) -%}
+    <button class="button is-primary is-small" onclick="{{ action }}">{{ text|safe }}</button>
+{%- endmacro -%}
+{%- macro get_button_tag(icon_name, method, visible=True, name="") -%}
+    <span class="{{ "tag is-link is-normal" if visible else "" }}" type="button" onclick="{{ method }}" {{ "" if visible else "hidden='true'" }} {{ "" if name=="" else "name='" + name +"'"}}>{{icon(icon_name)}}</span>
+{%- endmacro -%}
+{%- macro get_button_reset() -%}
+    <span class="tag is-link is-danger" type="button" onclick="setCheckBoxes(false)"><i class="mdi mdi-checkbox-multiple-blank"></i></span>
+{%- endmacro -%}
+{%- macro get_button_all() -%}
+    <span class="tag is-link is-success" type="button" onclick="setCheckBoxes(true)"><i class="mdi mdi-checkbox-multiple-marked"></i></span>
+{%- endmacro -%}
+{%- macro get_tag_class() -%}
+tag is-info is-small
+{%- endmacro -%}
+{%- macro get_container_class() -%}
+container is-fluid
+{%- endmacro -%}
+{%- macro selected() -%}
+is-selected
+{%- endmacro -%}
+{%- macro get_level_class() -%}
+level
+{%- endmacro -%}
+{%- macro get_align_right() -%}
+has-text-right
+{%- endmacro -%}
+{%- macro get_align_center() -%}
+has-text-center
+{%- endmacro -%}
+{%- macro get_left_position() -%}
+float-left
+{%- endmacro -%}
+{%- macro get_right_position() -%}
+float-right
+{%- endmacro -%}
+{%- macro get_row_head_class() -%}
+is-selected
+{%- endmacro -%}
+{%- macro get_align_center() -%}
+has-text-center
+{%- endmacro -%}

benchmark/scripts/app/templates/partials/datasets_table.html

@@ -0,0 +1,27 @@
+{% extends "base_" ~ framework ~ ".html" %}
+{% block content %}
+  <table id="file-table" class="{{ table_class }}">
+    <thead>
+      <tr class="{{ head_class }}">
+        <th class="{{ text_center }}">Dataset</th>
+        <th class="{{ text_center }}">Samples</th>
+        <th class="{{ text_center }}">Features</th>
+        <th class="{{ text_center }}">Cont. Feat.</th>
+        <th class="{{ text_center }}">Classes</th>
+        <th class="{{ text_center }}">Balance</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for dataset in datasets %}
+        <tr>
+          <td>{{ dataset.dataset }}</td>
+          <td class="{{ text_right }}">{{ "{:,}".format(dataset.samples) }}</td>
+          <td class="{{ text_right }}">{{ "{:,}".format(dataset.features) }}</td>
+          <td class="{{ text_right }}">{{ dataset.cont_features }}</td>
+          <td class="{{ text_right }}">{{ dataset.classes }}</td>
+          <td>{{ dataset.balance }}</td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+{% endblock %}

benchmark/scripts/app/templates/partials/table_report.html

@@ -0,0 +1,14 @@
+{% for item in data.results %}
+  <tr>
+    <td>{{ item.dataset }}</td>
+    <td class="{{ right }}">{{ '{:,}'.format(item.samples) }}</td>
+    <td class="{{ right }}">{{"%d" % item.features}}</td>
+    <td class="{{ right }}">{{"%d" % item.classes}}</td>
+    <td class="{{ right }}">{{ '{:,.2f}'.format(item.nodes|float) }}</td>
+    <td class="{{ right }}">{{ '{:,.2f}'.format(item.leaves|float) }}</td>
+    <td class="{{ right }}">{{ '{:,.2f}'.format(item.depth|float) }}</td>
+    <td class="{{ right }}">{{"%.6f±%.4f" % (item.score, item.score_std)}} {{ item.symbol|safe }}</td>
+    <td class="{{ right }}">{{"%.6f±%.4f" % (item.time, item.time_std)}}</td>
+    <td class="{{ center }}">{{ item.hyperparameters }}</td>
+  </tr>
+{% endfor %}

benchmark/scripts/app/templates/partials/table_report_bootstrap.html

@@ -0,0 +1,102 @@
+<div id="app">
+  <section class="section">
+    <div class="container-fluid">
+      <div class="p-4 bg-primary text-white">
+        <button type="button"
+                class="btn-close"
+                aria-label="Close"
+                onclick="location.href = '{{ back }}'"></button>
+        <h1>{{ data.title }}</h1>
+      </div>
+      <div>
+        <table class="table table-bordered">
+          <thead>
+            <tr class="bg-info text-white">
+              <th class="text-center">Platform</th>
+              <th class="text-center">Model</th>
+              <th class="text-center">Date</th>
+              <th class="text-center">Time</th>
+              {% if data.duration > 7200 %}
+                {% set unit = "h" %}
+                {% set divider = 3600 %}
+              {% else %}
+                {% set unit = "min" %}
+                {% set divider = 60 %}
+              {% endif %}
+              <th class="text-center">Duration ({{ unit }})</th>
+              <th class="text-center">Stratified</th>
+              <th class="text-center">Discretized</th>
+              <th class="text-center"># Folds</th>
+            </tr>
+            <tr>
+              <th class="text-center">{{ data.platform }}</th>
+              <th class="text-center">{{ data.model }} {{ data.version }}</th>
+              <th class="text-center">{{ data.date }}</th>
+              <th class="text-center">{{ data.time }}</th>
+              <th class="text-center">{{ "%.2f" % (data.duration/divider) }}</th>
+              <th class="text-center">{{ data.stratified }}</th>
+              <th class="text-center">{{ data.discretized }}</th>
+              <th class="text-center">{{ data.folds }}</th>
+            </tr>
+            <tr>
+              <th class="text-center bg-info text-white">Language</th>
+              <th class="text-center" colspan=3>{{ data.language }} {{ data.language_version }}</th>
+              <th class="text-center bg-info text-white">Seeds</th>
+              <th class="text-center" colspan=6>{{ data.seeds }}</th>
+            </tr>
+          </thead>
+        </table>
+        <div>
+          <button class="{{ button }}" onclick="excelFile()">
+            <i class="mdi mdi-file-excel"></i> Excel
+          </button>
+        </div>
+        <table id="report-table"
+               class="table table-striped table-hover table-bordered">
+          <thead>
+            <tr class="bg-primary text-white">
+              <th class="text-center">Dataset</th>
+              <th class="text-center">Samples</th>
+              <th class="text-center">Features</th>
+              <th class="text-center">Classes</th>
+              <th class="text-center">{{ app_config.nodes }}</th>
+              <th class="text-center">{{ app_config.leaves }}</th>
+              <th class="text-center">{{ app_config.depth }}</th>
+              <th class="text-center">{{ data.score_name|capitalize }}</th>
+              <th class="text-center">Time</th>
+              <th class="text-center">hyperparameters</th>
+            </tr>
+          </thead>
+          <tbody>
+            {% include "partials/table_report.html" %}
+          </tbody>
+        </table>
+        {% if summary|length > 0 %}
+          <div class="col-4 col-lg-4">
+            <table class="table table-bordered">
+              <thead>
+                <tr>
+                  <th class="text-center bg-primary text-white">Symbol</th>
+                  <th class="text-center bg-primary text-white">Meaning</th>
+                  <th class="text-center bg-primary text-white">Count</th>
+                </tr>
+              </thead>
+              {% include "partials/table_summary.html" %}
+            </table>
+          </div>
+        {% endif %}
+        <button type="button"
+                class="btn-close"
+                aria-label="Close"
+                onclick="location.href = '{{ back }}'"></button>
+        <h7>
+        <b>
+          Total score: {{ "%.6f" % (data.results | sum(attribute="score") ) }}
+        </b>
+        </h7>
+        <h7>
+        Number of files: {{ data.results | length }}
+        </h7>
+      </div>
+    </section>
+  </div>

benchmark/scripts/app/templates/partials/table_report_bulma.html

@@ -0,0 +1,100 @@
+<div id="app">
+  <header>
+    <div class="container is-fluid">
+      <div class="hero is-info is-bold">
+        <div class="hero-body">
+          <button class="delete is-large" onclick="location.href = '{{ back }}'"></button>
+          <h1 class="is-size-3">{{ data.title }}</h1>
+        </div>
+      </div>
+    </div>
+  </header>
+  <section class="section">
+    <div class="container is-fluid">
+      <div>
+        <table class="table is-fullwidth is-striped is-bordered">
+          <thead>
+            <tr class="is-selected">
+              <th class="has-text-centered">Platform</th>
+              <th class="has-text-centered">Model</th>
+              <th class="has-text-centered">Date</th>
+              <th class="has-text-centered">Time</th>
+              {% if data.duration > 7200 %}
+                {% set unit = "h" %}
+                {% set divider = 3600 %}
+              {% else %}
+                {% set unit = "min" %}
+                {% set divider = 60 %}
+              {% endif %}
+              <th class="has-text-centered">Duration ({{ unit }})</th>
+              <th class="has-text-centered">Stratified</th>
+              <th class="has-text-centered">Discretized</th>
+              <th class="has-text-centered"># Folds</th>
+            </tr>
+            <tr>
+              <th class="has-text-centered">{{ data.platform }}</th>
+              <th class="has-text-centered">{{ data.model }} {{ data.version }}</th>
+              <th class="has-text-centered">{{ data.date }}</th>
+              <th class="has-text-centered">{{ data.time }}</th>
+              <th class="has-text-centered">{{ "%.2f" % (data.duration/divider) }}</th>
+              <th class="has-text-centered">{{ data.stratified }}</th>
+              <th class="has-text-centered">{{ data.discretized }}</th>
+              <th class="has-text-centered">{{ data.folds }}</th>
+            </tr>
+            <tr>
+              <th class="has-text-center is-selected">Language</th>
+              <th class="has-text-centered" colspan=3>{{ data.language }} {{ data.language_version }}</th>
+              <th class="has-text-centered is-selected">Seeds</th>
+              <th class="has-text-centered" colspan=6>{{ data.seeds }}</th>
+            </tr>
+          </thead>
+        </table>
+        <div>
+          <button class="{{ button }}" onclick="excelFile()">
+            <i class="mdi mdi-file-excel"></i> Excel
+          </button>
+        </div>
+        <table id="report-table"
+               class="table is-fullwidth is-striped is-hoverable is-bordered">
+          <thead>
+            <tr class="is-selected">
+              <th class="has-text-centered">Dataset</th>
+              <th class="has-text-centered">Samples</th>
+              <th class="has-text-centered">Features</th>
+              <th class="has-text-centered">Classes</th>
+              <th class="has-text-centered">{{ app_config.nodes }}</th>
+              <th class="has-text-centered">{{ app_config.leaves }}</th>
+              <th class="has-text-centered">{{ app_config.depth }}</th>
+              <th class="has-text-centered">{{ data.score_name|capitalize }}</th>
+              <th class="has-text-centered">Time</th>
+              <th class="has-text-centered">hyperparameters</th>
+            </tr>
+          </thead>
+          <tbody>
+            {% include "partials/table_report.html" %}
+          </tbody>
+        </table>
+        {% if summary|length > 0 %}
+          <div class="col-2 col-lg-2">
+            <table class="table is-bordered">
+              <thead>
+                <tr class="is-selected">
+                  <th class="has-text-centered">Symbol</th>
+                  <th class="has-text-centered">Meaning</th>
+                  <th class="has-text-centered">Count</th>
+                </tr>
+              </thead>
+              {% include "partials/table_summary.html" %}
+            </table>
+          </div>
+        {% endif %}
+        <h2 class="has-text-white has-background-primary">
+          <b>
+            <button class="delete" onclick="location.href = '{{ back }}'"></button>
+            Total score: {{ "%.6f" % (data.results | sum(attribute="score") ) }}
+          </b>
+        </h2>
+        <h2>Number of files: {{ data.results | length }}</h2>
+      </div>
+    </section>
+  </div>

benchmark/scripts/app/templates/partials/table_select.html

@@ -0,0 +1,41 @@
+<table id="file-table" class="{{ select.get_table_class() }}">
+  <thead>
+    <tr>
+      <th>Model</th>
+      <th>Metric</th>
+      <th>Platform</th>
+      <th>Date</th>
+      <th>Time</th>
+      <th>Stratified</th>
+      <th>Title</th>
+      <th>Score</th>
+      <th>{{ select.get_button_reset()|safe }} {{ select.get_button_all()|safe }}</th>
+    </tr>
+  </thead>
+  <tbody>
+    {% for file, data in files.items() %} 
+      {% set parts = file.split('_') %} 
+      {% set stratified = parts[6].split('.')[0] %}
+      <tr id="{{ file }}">
+        <td>{{ parts[2] }}</td>
+        <td>{{ parts[1] }}</td>
+        <td>{{ parts[3] }}</td>
+        <td>{{ parts[4] }}</td>
+        <td>{{ parts[5] }}</td>
+        <td>{{ 'True' if stratified =='1' else 'False' }}</td>
+        <td>{{ "%s" % data["title"] }}</td>
+        <td class="{{ select.get_align_right() }}">{{ "%.6f" % data["score"] }}</td>
+        <td>
+          {{ select.get_button_tag("table-eye", "showFile('" ~ file ~ "')") | safe }}
+          {% set file_best = "best_results_" ~ parts[1] ~ "_" ~ parts[2] ~ ".json" %}
+          {{ select.get_button_tag("star-circle-outline", "redirectDouble('best_results', '" ~ file_best ~ "')", visible=False, name="best_buttons") | safe }}
+          <input
+            type="checkbox"
+            name="selected_files"
+            value="{{ file }}"
+          />
+        </td>
+      </tr>
+    {% endfor %}
+  </tbody>
+</table>

benchmark/scripts/app/templates/partials/table_select_design.html

@@ -0,0 +1,15 @@
+<div class="{{ select.get_container_class() }}">
+    {{ select.header("Benchmark Results") }}
+    <div class="{{ select.get_level_class() }}">
+        <div class="{{ select.get_left_position() }}">
+            {{ select.get_button("Use " ~ candidate, "redirectDouble('config', '" ~ candidate ~ "')")|safe }}
+            {{ select.get_button(select.icon("excel") ~ " Excel", "excel()")|safe }}
+            {{ select.get_button(select.icon("database-eye") ~ " Datasets", "redirectSimple('datasets')")|safe }}
+        </div>
+        <div class="{{ select.get_right_position() }}">
+            <input type="checkbox" id="compare" name="compare" {% if compare %} {{ "checked" }} {% endif %}>
+            <span class="{{ select.get_tag_class() }}">Comparing with best results</span>
+        </div>
+    </div>
+    {% include "partials/table_select.html" %}
+</div>

benchmark/scripts/app/templates/partials/table_summary.html

@@ -0,0 +1,13 @@
+{% for key, value in summary.items() %}
+    <tr>
+        <td class="{{ center }}">
+            {{key}}
+        </td>
+        <td >
+            {{value[0]}}
+        </td>
+        <td class={{ right }}>
+            {{'{:,}'.format(value[1])}}
+        </td>
+    </tr>
+{% endfor %}

benchmark/scripts/app/templates/report.html

@@ -0,0 +1,29 @@
+{% macro javascript(file) %}
+  <script src="{{ url_for('static', filename=file) }}"></script>
+{% endmacro %}
+{% set title = 'Report Viewer' %}
+{% extends 'base_' ~ framework ~ '.html' %}
+{% block content %}
+  {% if framework == 'bootstrap' %}
+    {% set center = 'text-center' %}
+    {% set right = 'text-end' %}
+    {% set button = 'btn btn-primary' %}
+    {% include 'partials/table_report_bootstrap.html' %}
+  {% else %}
+    {% set center = 'has-text-centered' %}
+    {% set right = 'has-text-right' %}
+    {% set button = 'button is-primary' %}
+    {% include 'partials/table_report_bulma.html' %}
+  {% endif %}
+{% endblock %}
+{% block jscript %}
+  {% if framework == 'bootstrap' %}
+    {% set selected = 'selected' %}
+  {% else %}
+    {% set selected = 'is-selected' %}
+  {% endif %}
+  <script>
+    {% include "js/report.js" %}
+  </script>
+  {{ javascript("js/excelFiles.js") }}
+{% endblock %}

benchmark/scripts/app/templates/report_best.html

@@ -0,0 +1,47 @@
+{% set title = "Best Results" %}
+{% extends "base_" ~ framework ~ ".html" %}
+{% import "partials/cfg_select_" ~ framework ~ ".jinja" as select %}
+{% block content %}
+    <div class="container">
+        {{ select.header(title, True, url_for("main.index", compare=compare)) }}
+        <table id="file-table" class="{{ select.get_table_class() }}">
+            <thead>
+                <tr class="{{ select.get_row_head_class() }}">
+                <th class="{{ select.get_align_center() }}">Dataset</th>
+                <th class="{{ select.get_align_center() }}">Score</th>
+                <th class="{{ select.get_align_center() }}">Hyperparameters</th>
+                <th class="{{ select.get_align_center() }}">File</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for dataset, info in data.items() %}
+                <tr>
+                    <td>{{ dataset }}</td>
+                    <td class="{{ select.get_align_right() }}">{{ '%9.7f' % info[0] }}</td>
+                    <td class="{{ select.get_align_center() }}">{{ info[1] }}</td>
+                    <td>
+                    {% set url = url_for(request.endpoint, **request.view_args)|urlencode %}
+                    <a href="{{ url_for('main.showfile', file_name = info[2], compare = compare) }}?url={{ url }}">{{ info[2] }}</a>
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+{% endblock %}
+{% block jscript %}
+<script>
+    $(document).ready(function () {
+        // Check if row is selected
+        $('#file-table tbody').on('click', 'tr', function () {
+            if ($(this).hasClass('{{ select.selected() }}')) {
+                $(this).removeClass('{{ select.selected() }}');
+            } else {
+                $('#file-table tbody tr.{{ select.selected() }}').removeClass("{{ select.selected() }}")
+                $(this).addClass('{{ select.selected() }}');
+            }
+        });
+  });
+</script>
+{% endblock %}
+

benchmark/scripts/app/templates/select.html

@@ -0,0 +1,20 @@
+{% macro javascript(file) %}
+  <script src="{{ url_for('static', filename=file) }}"></script>
+{% endmacro %}
+{% set title = 'Benchmark Results' %}
+{% extends 'base_' ~ framework ~ '.html' %}
+{% import 'partials/cfg_select_' ~ framework ~ '.jinja' as select %}
+{% block content %}
+  {% include 'partials/table_select_design.html' %}
+{% endblock %}
+
+{% block jscript %}
+  <script src="https://cdn.datatables.net/1.10.25/js/jquery.dataTables.min.js"></script>
+  {% if framework == 'bootstrap' %}
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.3/dist/js/bootstrap.min.js" integrity="sha384-cuYeSxntonz0PPNlHhBs68uyIAVpIIOZZ5JqeqvYYIcEL727kskC66kF92t6Xl2V" crossorigin="anonymous"></script>
+  {% endif %}
+  <script>
+    {% include'/js/select.js' %}
+  </script>
+  {{ javascript('js/excelFiles.js') }}
+{% endblock %}

benchmark/scripts/be_benchmark.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
-from benchmark.Results import Benchmark
+from benchmark.ResultsFiles import Benchmark
 from benchmark.Utils import Files
 from benchmark.Arguments import Arguments
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    arguments = Arguments(prog="be_benchmark")
     arguments.xset("score").xset("excel").xset("tex_output").xset("quiet")
     args = arguments.parse(args_test)
     benchmark = Benchmark(score=args.score, visualize=not args.quiet)

benchmark/scripts/be_best.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 import json
-from benchmark.Results import Summary
+from benchmark.ResultsBase import Summary
 from benchmark.Arguments import ALL_METRICS, Arguments
 
 

benchmark/scripts/be_build_best.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from benchmark.Results import ReportBest
-from benchmark.Experiments import Datasets, BestResults
+from benchmark.Experiments import BestResults
+from benchmark.Datasets import Datasets
 from benchmark.Arguments import Arguments
 
 """Build a json file with the best results of a model and its hyperparameters
@@ -20,5 +21,5 @@ def main(args_test=None):
         print(e)
     else:
         if args.report:
-            report = ReportBest(args.score, args.model, best=True, grid=False)
+            report = ReportBest(args.score, args.model, best=True)
             report.report()

benchmark/scripts/be_build_grid.py

@@ -46,7 +46,7 @@ def main(args_test=None):
         '{"C": 7, "gamma": 0.1, "kernel": "rbf", "multiclass_strategy": '
         '"ovr"}',
         '{"C": 5, "kernel": "rbf", "gamma": "auto"}',
-        '{"C": 0.05, "max_iter": 10000.0, "kernel": "liblinear", '
+        '{"C": 0.05, "max_iter": 10000, "kernel": "liblinear", '
         '"multiclass_strategy": "ovr"}',
         '{"C":0.0275, "kernel": "liblinear", "multiclass_strategy": "ovr"}',
         '{"C": 7, "gamma": 10.0, "kernel": "rbf", "multiclass_strategy": '
@@ -97,7 +97,7 @@ def main(args_test=None):
     for item in results:
         results_tmp = {"n_jobs": [-1], "n_estimators": [100]}
         for key, value in results[item].items():
-            new_key = f"base_estimator__{key}"
+            new_key = f"estimator__{key}"
             try:
                 results_tmp[new_key] = sorted(value)
             except TypeError:
@@ -111,6 +111,7 @@ def main(args_test=None):
                 t2 = sorted([x for x in value if isinstance(x, str)])
                 results_tmp[new_key] = t1 + t2
         output.append(results_tmp)
+
     # save results
     file_name = Files.grid_input(args.score, args.model)
     file_output = os.path.join(Folders.results, file_name)

benchmark/scripts/be_flask.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+import webbrowser
+from benchmark.Arguments import Arguments
+from benchmark.scripts.app.app import create_app, TEST, OUTPUT
+
+# Launch a flask server to serve the results
+
+def main(args_test=None):
+    arguments = Arguments(prog="be_flask")
+    arguments.xset("output")
+    args = arguments.parse(args_test)
+    app = create_app()
+    app.config[TEST] = args_test is not None
+    app.config[OUTPUT] = args.output
+    print("Output is ", args.output)
+    if args.output == "local":
+        webbrowser.open_new("http://127.0.0.1:1234/")
+    app.run(port=1234, host="0.0.0.0")

benchmark/scripts/be_grid.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-from benchmark.Experiments import GridSearch, Datasets
+from benchmark.Experiments import GridSearch
+from benchmark.Datasets import Datasets
 from benchmark.Arguments import Arguments
 
 """Do experiment and build result file, optionally print report with results

benchmark/scripts/be_init_project.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+import os
+from benchmark.Utils import Files, Folders
+from benchmark.Arguments import Arguments
+
+
+def main(args_test=None):
+    arguments = Arguments(prog="be_init_project")
+    arguments.add_argument("project_name", help="Project name")
+    args = arguments.parse(args_test)
+    folders = []
+    folders.append(args.project_name)
+    folders.append(os.path.join(args.project_name, Folders.results))
+    folders.append(os.path.join(args.project_name, Folders.hidden_results))
+    folders.append(os.path.join(args.project_name, Folders.exreport))
+    folders.append(os.path.join(args.project_name, Folders.report))
+    folders.append(os.path.join(args.project_name, Folders.img))
+    folders.append(os.path.join(args.project_name, Folders.excel))
+    folders.append(os.path.join(args.project_name, Folders.sql))
+
+    try:
+        for folder in folders:
+            print(f"Creating folder {folder}")
+            os.makedirs(folder)
+    except FileExistsError as e:
+        print(e)
+        exit(1)
+    env_src = os.path.join(Folders.src(), "..", f"{Files.dot_env}.dist")
+    env_to = os.path.join(args.project_name, Files.dot_env)
+    os.system(f"cp {env_src} {env_to}")
+    print("Done!")
+    print(
+        "Please, edit .env file with your settings and add a datasets folder"
+    )
+    print("with an all.txt file with the datasets you want to use.")
+    print("In that folder you have to include all the datasets you'll use.")

benchmark/scripts/be_list.py

@@ -1,19 +1,21 @@
 #! /usr/bin/env python
 import os
-from benchmark.Results import Summary
-from benchmark.Utils import Folders
+from benchmark.ResultsBase import Summary
+from benchmark.Utils import Files, Folders
 from benchmark.Arguments import Arguments
+from benchmark.Manager import Manage
 
 """List experiments of a model
 """
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    arguments = Arguments(prog="be_list")
     arguments.xset("number").xset("model", required=False).xset("key")
-    arguments.xset("hidden").xset("nan").xset("score", required=False)
+    arguments.xset("score", required=False).xset("compare").xset("hidden")
+    arguments.xset("nan")
     args = arguments.parse(args_test)
-    data = Summary(hidden=args.hidden)
+    data = Summary(hidden=args.hidden, compare=args.compare)
     data.acquire()
     try:
         data.list_results(
@@ -21,33 +23,14 @@ def main(args_test=None):
             model=args.model,
             sort_key=args.key,
             number=args.number,
+            nan=args.nan,
         )
     except ValueError as e:
         print(e)
-    else:
-        if args.nan:
-            results_nan = []
-            results = data.get_results_criteria(
-                score=args.score,
-                model=args.model,
-                input_data=None,
-                sort_key=args.key,
-                number=args.number,
-            )
-            for result in results:
-                if result["metric"] != result["metric"]:
-                    results_nan.append(result)
-            if results_nan != []:
-                print(
-                    "\n"
-                    + "*" * 30
-                    + " Results with nan moved to hidden "
-                    + "*" * 30
-                )
-                data.list_results(input_data=results_nan)
-                for result in results_nan:
-                    name = result["file"]
-                    os.rename(
-                        os.path.join(Folders.results, name),
-                        os.path.join(Folders.hidden_results, name),
-                    )
+        return
+    manager = Manage(data)
+    excel_generated = manager.manage_results()
+    if excel_generated:
+        name = os.path.join(Folders.excel, Files.be_list_excel)
+        print(f"Generated file: {name}")
+        Files.open(name, test=args_test is not None)

benchmark/scripts/be_main.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import os
-from benchmark.Experiments import Experiment, Datasets
+from benchmark.Experiments import Experiment
+from benchmark.Datasets import Datasets
 from benchmark.Results import Report
 from benchmark.Arguments import Arguments
 
@@ -9,28 +10,39 @@ from benchmark.Arguments import Arguments
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    arguments = Arguments(prog="be_main")
     arguments.xset("stratified").xset("score").xset("model", mandatory=True)
     arguments.xset("n_folds").xset("platform").xset("quiet").xset("title")
-    arguments.xset("hyperparameters").xset("paramfile").xset("report")
-    arguments.xset("grid_paramfile").xset("dataset")
+    arguments.xset("report").xset("ignore_nan").xset("discretize")
+    arguments.xset("fit_features")
+    arguments.add_exclusive(
+        ["grid_paramfile", "best_paramfile", "hyperparameters"]
+    )
+    arguments.xset(
+        "dataset", overrides="title", const="Test with only one dataset"
+    )
     args = arguments.parse(args_test)
     report = args.report or args.dataset is not None
     if args.grid_paramfile:
-        args.paramfile = False
+        args.best_paramfile = False
     try:
         job = Experiment(
             score_name=args.score,
             model_name=args.model,
             stratified=args.stratified,
-            datasets=Datasets(dataset_name=args.dataset),
+            datasets=Datasets(
+                dataset_name=args.dataset, discretize=args.discretize
+            ),
             hyperparams_dict=args.hyperparameters,
-            hyperparams_file=args.paramfile,
+            hyperparams_file=args.best_paramfile,
             grid_paramfile=args.grid_paramfile,
             progress_bar=not args.quiet,
             platform=args.platform,
+            ignore_nan=args.ignore_nan,
             title=args.title,
             folds=args.n_folds,
+            fit_features=args.fit_features,
+            discretize=args.discretize,
         )
         job.do_experiment()
     except ValueError as e:

benchmark/scripts/be_print_strees.py

@@ -3,7 +3,7 @@ import os
 import json
 from stree import Stree
 from graphviz import Source
-from benchmark.Experiments import Datasets
+from benchmark.Datasets import Datasets
 from benchmark.Utils import Files, Folders
 from benchmark.Arguments import Arguments
 

benchmark/scripts/be_report.py

@@ -1,46 +1,88 @@
 #!/usr/bin/env python
-from benchmark.Results import Report, Excel, SQL, ReportBest, ReportDatasets
-from benchmark.Utils import Files
+import os
+from benchmark.Results import Report, ReportBest
+from benchmark.ResultsFiles import Excel, SQLFile, ReportDatasets
+from benchmark.Utils import Files, Folders
 from benchmark.Arguments import Arguments
+from pathlib import Path
 
 
 """Build report on screen of a result file, optionally generate excel and sql
-file, and can compare results of report with best results obtained by model
+file, and can compare results of report wibth best results obtained by model
 If no argument is set, displays the datasets and its characteristics
 """
 
 
 def main(args_test=None):
-    arguments = Arguments()
-    arguments.xset("file").xset("excel").xset("sql").xset("compare")
-    arguments.xset("best").xset("grid").xset("model", required=False)
-    arguments.xset("score", required=False)
+    is_test = args_test is not None
+    arguments = Arguments(prog="be_report")
+    arguments.add_subparser()
+    arguments.add_subparsers_options(
+        (
+            "best",
+            "Report best results obtained by any model/score. "
+            "See be_build_best",
+        ),
+        [
+            ("model", dict(required=False)),
+            ("score", dict(required=False)),
+        ],
+    )
+    arguments.add_subparsers_options(
+        (
+            "grid",
+            "Report grid results obtained by any model/score. "
+            "See be_build_grid",
+        ),
+        [
+            ("model", dict(required=False)),
+            ("score", dict(required=False)),
+        ],
+    )
+    arguments.add_subparsers_options(
+        ("file", "Report file results"),
+        [
+            ("file_name", {}),
+            ("excel", {}),
+            ("sql", {}),
+            ("compare", {}),
+        ],
+    )
+    arguments.add_subparsers_options(
+        ("datasets", "Report datasets information"),
+        [
+            ("excel", {}),
+        ],
+    )
     args = arguments.parse(args_test)
-    if args.best:
-        args.grid = None
-    if args.grid:
-        args.best = None
-    if args.file is None and args.best is None and args.grid is None:
-        ReportDatasets.report()
-    else:
-        if args.best is not None or args.grid is not None:
-            report = ReportBest(args.score, args.model, args.best, args.grid)
+    match args.subcommand:
+        case "best" | "grid":
+            best = args.subcommand == "best"
+            report = ReportBest(args.score, args.model, best)
             report.report()
-        else:
+        case "file":
             try:
-                report = Report(args.file, args.compare)
+                report = Report(args.file_name, args.compare)
+                report.report()
             except FileNotFoundError as e:
                 print(e)
-            else:
-                report.report()
-                if args.excel:
-                    excel = Excel(
-                        file_name=args.file,
-                        compare=args.compare,
-                    )
-                    excel.report()
-                    is_test = args_test is not None
-                    Files.open(excel.get_file_name(), is_test)
-                if args.sql:
-                    sql = SQL(args.file)
-                    sql.report()
+                return
+            if args.sql:
+                sql = SQLFile(args.file_name)
+                sql.report()
+            if args.excel:
+                excel = Excel(
+                    file_name=Path(args.file_name).name,
+                    compare=args.compare,
+                )
+                excel.report()
+                Files.open(
+                    os.path.join(Folders.excel, excel.get_file_name()), is_test
+                )
+        case "datasets":
+            report = ReportDatasets(args.excel)
+            report.report()
+            if args.excel:
+                Files.open(report.get_file_name(), is_test)
+        case _:
+            arguments.print_help()

benchmark/scripts/be_summary.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from benchmark.Results import Summary
+from benchmark.ResultsBase import Summary
 from benchmark.Arguments import ALL_METRICS, Arguments
 
 

benchmark/tests/.env

@@ -5,3 +5,10 @@ model=ODTE
 stratified=0
 # Source of data Tanveer/Surcov
 source_data=Tanveer
+seeds=[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]
+discretize=0
+nodes=Nodes
+leaves=Leaves
+depth=Depth
+fit_features=0
+margin=0.1

benchmark/tests/.env.arff

@@ -0,0 +1,13 @@
+score=accuracy
+platform=MacBookpro16
+n_folds=5
+model=ODTE
+stratified=0
+source_data=Arff
+seeds=[271, 314, 171]
+discretize=1
+nodes=Nodes
+leaves=Leaves
+depth=Depth
+fit_features=1
+margin=0.1

benchmark/tests/.env.dist

@@ -5,3 +5,10 @@ model=ODTE
 stratified=0
 # Source of data Tanveer/Surcov
 source_data=Tanveer
+seeds=[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]
+discretize=0
+nodes=Nodes
+leaves=Leaves
+depth=Depth
+fit_features=0
+margin=0.1

benchmark/tests/.env.surcov

@@ -5,3 +5,10 @@ model=ODTE
 stratified=0
 # Source of data Tanveer/Surcov
 source_data=Surcov
+seeds=[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]
+discretize=0
+nodes=Nodes
+leaves=Leaves
+depth=Depth
+fit_features=0
+margin=0.1

benchmark/tests/.gitignore

@@ -0,0 +1,2 @@
+ReportDatasets.xlsx 
+some_results.xlsx

benchmark/tests/Arguments_test.py

@@ -24,13 +24,11 @@ class ArgumentsTest(TestBase):
 
     def test_parameters(self):
         expected_parameters = {
-            "best": ("-b", "--best"),
+            "best_paramfile": ("-b", "--best_paramfile"),
             "color": ("-c", "--color"),
             "compare": ("-c", "--compare"),
             "dataset": ("-d", "--dataset"),
             "excel": ("-x", "--excel"),
-            "file": ("-f", "--file"),
-            "grid": ("-g", "--grid"),
             "grid_paramfile": ("-g", "--grid_paramfile"),
             "hidden": ("--hidden",),
             "hyperparameters": ("-p", "--hyperparameters"),
@@ -42,7 +40,6 @@ class ArgumentsTest(TestBase):
             "nan": ("--nan",),
             "number": ("-n", "--number"),
             "n_folds": ("-n", "--n_folds"),
-            "paramfile": ("-f", "--paramfile"),
             "platform": ("-P", "--platform"),
             "quiet": ("-q", "--quiet"),
             "report": ("-r", "--report"),
@@ -71,7 +68,7 @@ class ArgumentsTest(TestBase):
         test_args = ["-n", "3", "-k", "date"]
         with self.assertRaises(SystemExit):
             arguments.parse(test_args)
-        self.assertRegexpMatches(
+        self.assertRegex(
             stderr.getvalue(),
             r"error: the following arguments are required: -m/--model",
         )
@@ -82,7 +79,7 @@ class ArgumentsTest(TestBase):
         test_args = ["-n", "3", "-m", "SVC"]
         with self.assertRaises(SystemExit):
             arguments.parse(test_args)
-        self.assertRegexpMatches(
+        self.assertRegex(
             stderr.getvalue(),
             r"error: the following arguments are required: -k/--key",
         )
@@ -98,3 +95,27 @@ class ArgumentsTest(TestBase):
         finally:
             os.chdir(path)
         self.assertEqual(stderr.getvalue(), f"{NO_ENV}\n")
+
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_overrides(self, stderr):
+        arguments = self.build_args()
+        arguments.xset("title")
+        arguments.xset("dataset", overrides="title", const="sample text")
+        test_args = ["-n", "3", "-m", "SVC", "-k", "1", "-d", "dataset"]
+        args = arguments.parse(test_args)
+        self.assertEqual(stderr.getvalue(), "")
+        self.assertEqual(args.title, "sample text")
+
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_overrides_no_args(self, stderr):
+        arguments = self.build_args()
+        arguments.xset("title")
+        arguments.xset("dataset", overrides="title", const="sample text")
+        test_args = None
+        with self.assertRaises(SystemExit):
+            arguments.parse(test_args)
+        self.assertRegex(
+            stderr.getvalue(),
+            r"error: the following arguments are required: -m/--model, "
+            "-k/--key, --title",
+        )

benchmark/tests/Benchmark_test.py

@@ -4,7 +4,8 @@ from unittest.mock import patch
 from openpyxl import load_workbook
 from .TestBase import TestBase
 from ..Utils import Folders, Files, NO_RESULTS
-from ..Results import Benchmark
+from ..ResultsFiles import Benchmark
+from .._version import __version__
 
 
 class BenchmarkTest(TestBase):
@@ -14,10 +15,10 @@ class BenchmarkTest(TestBase):
             files.append(Files.exreport(score))
             files.append(Files.exreport_output(score))
             files.append(Files.exreport_err(score))
-        files.append(Files.exreport_excel(score))
         files.append(Files.exreport_pdf)
         files.append(Files.tex_output("accuracy"))
         self.remove_files(files, Folders.exreport)
+        self.remove_files([Files.exreport_excel("accuracy")], Folders.excel)
         self.remove_files(files, ".")
         return super().tearDown()
 
@@ -98,9 +99,16 @@ class BenchmarkTest(TestBase):
         benchmark.excel()
         file_name = benchmark.get_excel_file_name()
         book = load_workbook(file_name)
+        replace = None
+        with_this = None
         for sheet_name in book.sheetnames:
             sheet = book[sheet_name]
-            self.check_excel_sheet(sheet, f"exreport_excel_{sheet_name}")
-            # ExcelTest.generate_excel_sheet(
-            #     self, sheet, f"exreport_excel_{sheet_name}"
-            # )
+            if sheet_name == "Datasets":
+                replace = self.benchmark_version
+                with_this = __version__
+            self.check_excel_sheet(
+                sheet,
+                f"exreport_excel_{sheet_name}",
+                replace=replace,
+                with_this=with_this,
+            )

benchmark/tests/BestResults_test.py

@@ -1,6 +1,7 @@
 import os
 from .TestBase import TestBase
-from ..Experiments import BestResults, Datasets
+from ..Experiments import BestResults
+from ..Datasets import Datasets
 
 
 class BestResultTest(TestBase):
@@ -17,7 +18,7 @@ class BestResultTest(TestBase):
                     "C": 7,
                     "gamma": 0.1,
                     "kernel": "rbf",
-                    "max_iter": 10000.0,
+                    "max_iter": 10000,
                     "multiclass_strategy": "ovr",
                 },
                 "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",

benchmark/tests/Dataset_test.py

@@ -1,6 +1,6 @@
-import shutil
 from .TestBase import TestBase
-from ..Experiments import Randomized, Datasets
+from ..Experiments import Randomized
+from ..Datasets import Datasets
 
 
 class DatasetTest(TestBase):
@@ -16,18 +16,33 @@ class DatasetTest(TestBase):
         self.set_env(".env.dist")
         return super().tearDown()
 
-    @staticmethod
-    def set_env(env):
-        shutil.copy(env, ".env")
-
     def test_Randomized(self):
         expected = [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]
-        self.assertSequenceEqual(Randomized.seeds, expected)
+        self.assertSequenceEqual(Randomized.seeds(), expected)
+
+    def test_Randomized_3_seeds(self):
+        self.set_env(".env.arff")
+        expected = [271, 314, 171]
+        self.assertSequenceEqual(Randomized.seeds(), expected)
+
+    def test_load_dataframe(self):
+        self.set_env(".env.arff")
+        dt = Datasets()
+        X, y = dt.load("iris", dataframe=False)
+        dataset = dt.load("iris", dataframe=True)
+        class_name = dt.get_class_name()
+        features = dt.get_features()
+        self.assertListEqual(y.tolist(), dataset[class_name].tolist())
+        for i in range(len(features)):
+            self.assertListEqual(
+                X[:, i].tolist(), dataset[features[i]].tolist()
+            )
 
     def test_Datasets_iterator(self):
         test = {
             ".env.dist": ["balance-scale", "balloons"],
             ".env.surcov": ["iris", "wine"],
+            ".env.arff": ["iris", "wine"],
         }
         for key, value in test.items():
             self.set_env(key)
@@ -51,6 +66,11 @@ class DatasetTest(TestBase):
         self.assertSequenceEqual(X.shape, (625, 4))
         self.assertSequenceEqual(y.shape, (625,))
 
+    def test_create_with_unknown_dataset(self):
+        with self.assertRaises(ValueError) as msg:
+            Datasets("unknown")
+        self.assertEqual(str(msg.exception), "Unknown dataset: unknown")
+
     def test_load_unknown_dataset(self):
         dt = Datasets()
         with self.assertRaises(ValueError) as msg:
@@ -61,6 +81,7 @@ class DatasetTest(TestBase):
         test = {
             ".env.dist": "balloons",
             ".env.surcov": "wine",
+            ".env.arff": "iris",
         }
         for key, value in test.items():
             self.set_env(key)

benchmark/tests/Excel_test.py

@@ -2,8 +2,8 @@ import os
 from openpyxl import load_workbook
 from xlsxwriter import Workbook
 from .TestBase import TestBase
-from ..Results import Excel
-from ..Utils import Folders
+from ..ResultsFiles import Excel
+from ..Utils import Folders, Files
 
 
 class ExcelTest(TestBase):
@@ -13,7 +13,7 @@ class ExcelTest(TestBase):
             "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.xlsx",
             "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.xlsx",
         ]
-        self.remove_files(files, Folders.results)
+        self.remove_files(files, Folders.excel)
         return super().tearDown()
 
     def test_report_excel_compared(self):
@@ -21,7 +21,7 @@ class ExcelTest(TestBase):
         report = Excel(file_name, compare=True)
         report.report()
         file_output = report.get_file_name()
-        book = load_workbook(file_output)
+        book = load_workbook(os.path.join(Folders.excel, file_output))
         sheet = book["STree"]
         self.check_excel_sheet(sheet, "excel_compared")
 
@@ -30,14 +30,14 @@ class ExcelTest(TestBase):
         report = Excel(file_name, compare=False)
         report.report()
         file_output = report.get_file_name()
-        book = load_workbook(file_output)
+        book = load_workbook(os.path.join(Folders.excel, file_output))
         sheet = book["STree"]
         self.check_excel_sheet(sheet, "excel")
 
     def test_Excel_Add_sheet(self):
         file_name = "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json"
-        excel_file_name = file_name.replace(".json", ".xlsx")
-        book = Workbook(os.path.join(Folders.results, excel_file_name))
+        excel_file_name = file_name.replace(Files.report_ext, ".xlsx")
+        book = Workbook(os.path.join(Folders.excel, excel_file_name))
         excel = Excel(file_name=file_name, book=book)
         excel.report()
         report = Excel(
@@ -46,7 +46,7 @@ class ExcelTest(TestBase):
         )
         report.report()
         book.close()
-        book = load_workbook(os.path.join(Folders.results, excel_file_name))
+        book = load_workbook(os.path.join(Folders.excel, excel_file_name))
         sheet = book["STree"]
         self.check_excel_sheet(sheet, "excel_add_STree")
         sheet = book["ODTE"]

benchmark/tests/Experiment_test.py

@@ -1,16 +1,21 @@
 import json
+from io import StringIO
+from unittest.mock import patch
 from .TestBase import TestBase
-from ..Experiments import Experiment, Datasets
+from ..Experiments import Experiment
+from ..Datasets import Datasets
 
 
 class ExperimentTest(TestBase):
     def setUp(self):
         self.exp = self.build_exp()
 
-    def build_exp(self, hyperparams=False, grid=False):
+    def build_exp(
+        self, hyperparams=False, grid=False, model="STree", ignore_nan=False
+    ):
         params = {
             "score_name": "accuracy",
-            "model_name": "STree",
+            "model_name": model,
             "stratified": "0",
             "datasets": Datasets(),
             "hyperparams_dict": "{}",
@@ -20,6 +25,7 @@ class ExperimentTest(TestBase):
             "title": "Test",
             "progress_bar": False,
             "folds": 2,
+            "ignore_nan": ignore_nan,
         }
         return Experiment(**params)
 
@@ -30,6 +36,7 @@ class ExperimentTest(TestBase):
             ],
             ".",
         )
+        self.set_env(".env.dist")
         return super().tearDown()
 
     def test_build_hyperparams_file(self):
@@ -45,7 +52,7 @@ class ExperimentTest(TestBase):
                     "C": 7,
                     "gamma": 0.1,
                     "kernel": "rbf",
-                    "max_iter": 10000.0,
+                    "max_iter": 10000,
                     "multiclass_strategy": "ovr",
                 },
                 "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
@@ -88,7 +95,7 @@ class ExperimentTest(TestBase):
     def test_exception_n_fold_crossval(self):
         self.exp.do_experiment()
         with self.assertRaises(ValueError):
-            self.exp._n_fold_crossval([], [], {})
+            self.exp._n_fold_crossval("", [], [], {})
 
     def test_do_experiment(self):
         self.exp.do_experiment()
@@ -130,3 +137,42 @@ class ExperimentTest(TestBase):
         ):
             for key, value in expected_result.items():
                 self.assertEqual(computed_result[key], value)
+
+    def test_build_fit_parameters(self):
+        self.set_env(".env.arff")
+        expected = {
+            "state_names": {
+                "sepallength": [0, 1, 2],
+                "sepalwidth": [0, 1, 2, 3, 4, 5],
+                "petallength": [0, 1, 2, 3],
+                "petalwidth": [0, 1, 2],
+            },
+            "features": [
+                "sepallength",
+                "sepalwidth",
+                "petallength",
+                "petalwidth",
+            ],
+        }
+        exp = self.build_exp(model="TAN")
+        X, y = exp.datasets.load("iris")
+        computed = exp._build_fit_params("iris")
+        for key, value in expected["state_names"].items():
+            self.assertEqual(computed["state_names"][key], value)
+        for feature in expected["features"]:
+            self.assertIn(feature, computed["features"])
+        # Ask for states of a dataset that does not exist
+        computed = exp._build_fit_params("not_existing")
+        self.assertTrue("states" not in computed)
+
+    @patch("sys.stdout", new_callable=StringIO)
+    def test_experiment_with_nan_not_ignored(self, mock_output):
+        exp = self.build_exp(model="Mock")
+        self.assertRaises(ValueError, exp.do_experiment)
+        output_text = mock_output.getvalue().splitlines()
+        expected = "[      nan 0.8974359]"
+        self.assertEqual(expected, output_text[0])
+
+    def test_experiment_with_nan_ignored(self):
+        self.exp = self.build_exp(model="Mock", ignore_nan=True)
+        self.exp.do_experiment()

benchmark/tests/GridSearch_test.py

@@ -1,6 +1,7 @@
 import json
 from .TestBase import TestBase
-from ..Experiments import GridSearch, Datasets
+from ..Experiments import GridSearch
+from ..Datasets import Datasets
 
 
 class GridSearchTest(TestBase):
@@ -77,7 +78,9 @@ class GridSearchTest(TestBase):
                 "v. 1.2.4, Computed on Test on 2022-02-22 at 12:00:00 took 1s",
             ],
         }
-        self.assertSequenceEqual(data, expected)
+        for key, value in expected.items():
+            self.assertEqual(data[key][0], value[0])
+            self.assertDictEqual(data[key][1], value[1])
 
     def test_duration_message(self):
         expected = ["47.234s", "5.421m", "1.177h"]

benchmark/tests/Models_test.py

@@ -15,6 +15,8 @@ from odte import Odte
 from xgboost import XGBClassifier
 from .TestBase import TestBase
 from ..Models import Models
+import xgboost
+import sklearn
 
 
 class ModelTest(TestBase):
@@ -33,22 +35,54 @@ class ModelTest(TestBase):
         for key, value in test.items():
             self.assertIsInstance(Models.get_model(key), value)
 
+    def test_Models_version(self):
+        def ver_stree():
+            return "1.2.3"
+
+        def ver_wodt():
+            return "h.j.k"
+
+        def ver_odte():
+            return "4.5.6"
+
+        test = {
+            "STree": [ver_stree, "1.2.3"],
+            "Wodt": [ver_wodt, "h.j.k"],
+            "ODTE": [ver_odte, "4.5.6"],
+            "RandomForest": [None, "7.8.9"],
+            "BaggingStree": [None, "x.y.z"],
+            "AdaBoostStree": [None, "w.x.z"],
+            "XGBoost": [None, "10.11.12"],
+        }
+        for key, value in test.items():
+            clf = Models.get_model(key)
+            if key in ["STree", "Wodt", "ODTE"]:
+                clf.version = value[0]
+            elif key == "XGBoost":
+                xgboost.__version__ = value[1]
+            else:
+                sklearn.__version__ = value[1]
+            self.assertEqual(Models.get_version(key, clf), value[1])
+
+    def test_bogus_Model_Version(self):
+        self.assertEqual(Models.get_version("unknown", None), "Error")
+
     def test_BaggingStree(self):
         clf = Models.get_model("BaggingStree")
         self.assertIsInstance(clf, BaggingClassifier)
-        clf_base = clf.base_estimator
+        clf_base = clf.estimator
         self.assertIsInstance(clf_base, Stree)
 
     def test_BaggingWodt(self):
         clf = Models.get_model("BaggingWodt")
         self.assertIsInstance(clf, BaggingClassifier)
-        clf_base = clf.base_estimator
+        clf_base = clf.estimator
         self.assertIsInstance(clf_base, Wodt)
 
     def test_AdaBoostStree(self):
         clf = Models.get_model("AdaBoostStree")
         self.assertIsInstance(clf, AdaBoostClassifier)
-        clf_base = clf.base_estimator
+        clf_base = clf.estimator
         self.assertIsInstance(clf_base, Stree)
 
     def test_unknown_classifier(self):
@@ -68,7 +102,7 @@ class ModelTest(TestBase):
         test = {
             "STree": ((11, 6, 4), 1.0),
             "Wodt": ((303, 152, 50), 0.9382022471910112),
-            "ODTE": ((7.86, 4.43, 3.37), 1.0),
+            "ODTE": ((786, 443, 337), 1.0),
             "Cart": ((23, 12, 5), 1.0),
             "SVC": ((0, 0, 0), 0.7078651685393258),
             "RandomForest": ((21.3, 11, 5.26), 1.0),

benchmark/tests/Report_test.py

@@ -2,11 +2,17 @@ import os
 from io import StringIO
 from unittest.mock import patch
 from .TestBase import TestBase
-from ..Results import Report, BaseReport, ReportBest, ReportDatasets
+from ..Results import Report, ReportBest
+from ..ResultsFiles import ReportDatasets
+from ..ResultsBase import BaseReport
+from ..Manager import get_input
 from ..Utils import Symbols
 
 
 class ReportTest(TestBase):
+    def test_get_input(self):
+        self.assertEqual(get_input(is_test=True), "test")
+
     def test_BaseReport(self):
         with patch.multiple(BaseReport, __abstractmethods__=set()):
             file_name = os.path.join(
@@ -60,31 +66,63 @@ class ReportTest(TestBase):
         self.assertEqual(res, Symbols.better_best)
         res = report._compute_status("balloons", 1.0)
         self.assertEqual(res, Symbols.better_best)
+        report = Report(file_name=file_name)
+        with patch(self.output, new=StringIO()):
+            report.report()
+        res = report._compute_status("balloons", 0.99)
+        self.assertEqual(res, Symbols.upward_arrow)
+        report.margin = 0.9
+        res = report._compute_status("balloons", 0.99)
+        self.assertEqual(res, Symbols.cross)
+
+    def test_reportbase_compute_status(self):
+        with patch.multiple(BaseReport, __abstractmethods__=set()):
+            file_name = os.path.join(
+                "results",
+                "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
+            )
+            temp = BaseReport(file_name)
+            temp.compare = False
+            temp._compare_totals = {}
+            temp.score_name = "f1"
+            res = temp._compute_status("balloons", 0.99)
+            self.assertEqual(res, " ")
 
     def test_report_file_not_found(self):
         with self.assertRaises(FileNotFoundError):
             _ = Report("unknown_file")
 
     def test_report_best(self):
-        report = ReportBest("accuracy", "STree", best=True, grid=False)
+        report = ReportBest("accuracy", "STree", best=True)
         with patch(self.output, new=StringIO()) as stdout:
             report.report()
         self.check_output_file(stdout, "report_best")
 
     def test_report_grid(self):
-        report = ReportBest("accuracy", "STree", best=False, grid=True)
+        report = ReportBest("accuracy", "STree", best=False)
         with patch(self.output, new=StringIO()) as stdout:
             report.report()
-        self.check_output_file(stdout, "report_grid")
-
-    def test_report_best_both(self):
-        report = ReportBest("accuracy", "STree", best=True, grid=True)
-        with patch(self.output, new=StringIO()) as stdout:
-            report.report()
-        self.check_output_file(stdout, "report_best")
+        file_name = "report_grid.test"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read().splitlines()
+        output_text = stdout.getvalue().splitlines()
+        # Compare replacing STree version
+        for line, index in zip(expected, range(len(expected))):
+            if self.stree_version in line:
+                # replace STree version
+                line = self.replace_STree_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])
 
     @patch("sys.stdout", new_callable=StringIO)
     def test_report_datasets(self, mock_output):
         report = ReportDatasets()
         report.report()
-        self.check_output_file(mock_output, "report_datasets")
+        file_name = f"report_datasets{self.ext}"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        output_text = mock_output.getvalue().splitlines()
+        for line, index in zip(expected.splitlines(), range(len(expected))):
+            if self.benchmark_version in line:
+                # replace benchmark version
+                line = self.replace_benchmark_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])

benchmark/tests/SQL_test.py

@@ -1,7 +1,7 @@
 import os
 from .TestBase import TestBase
-from ..Results import SQL
-from ..Utils import Folders
+from ..ResultsFiles import SQLFile
+from ..Utils import Folders, Files
 
 
 class SQLTest(TestBase):
@@ -9,14 +9,14 @@ class SQLTest(TestBase):
         files = [
             "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.sql",
         ]
-        self.remove_files(files, Folders.results)
+        self.remove_files(files, Folders.sql)
         return super().tearDown()
 
     def test_report_SQL(self):
         file_name = "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json"
-        report = SQL(file_name)
+        report = SQLFile(file_name)
         report.report()
         file_name = os.path.join(
-            Folders.results, file_name.replace(".json", ".sql")
+            Folders.sql, file_name.replace(Files.report_ext, ".sql")
         )
         self.check_file_file(file_name, "sql")

benchmark/tests/Summary_test.py

@@ -1,7 +1,7 @@
 from io import StringIO
 from unittest.mock import patch
 from .TestBase import TestBase
-from ..Results import Summary
+from ..ResultsBase import Summary
 from ..Utils import NO_RESULTS
 
 

benchmark/tests/TestBase.py

@@ -4,6 +4,7 @@ import pathlib
 import sys
 import csv
 import unittest
+import shutil
 from importlib import import_module
 from io import StringIO
 from unittest.mock import patch
@@ -14,8 +15,15 @@ class TestBase(unittest.TestCase):
         os.chdir(os.path.dirname(os.path.abspath(__file__)))
         self.test_files = "test_files"
         self.output = "sys.stdout"
+        self.ext = ".test"
+        self.benchmark_version = "0.2.0"
+        self.stree_version = "1.2.4"
         super().__init__(*args, **kwargs)
 
+    @staticmethod
+    def set_env(env):
+        shutil.copy(env, ".env")
+
     def remove_files(self, files, folder):
         for file_name in files:
             file_name = os.path.join(folder, file_name)
@@ -23,6 +31,7 @@ class TestBase(unittest.TestCase):
                 os.remove(file_name)
 
     def generate_excel_sheet(self, sheet, file_name):
+        file_name += self.ext
         with open(os.path.join(self.test_files, file_name), "w") as f:
             for row in range(1, sheet.max_row + 1):
                 for col in range(1, sheet.max_column + 1):
@@ -30,8 +39,10 @@ class TestBase(unittest.TestCase):
                     if value is not None:
                         print(f'{row};{col};"{value}"', file=f)
 
-    def check_excel_sheet(self, sheet, file_name):
-        file_name += ".test"
+    def check_excel_sheet(
+        self, sheet, file_name, replace=None, with_this=None
+    ):
+        file_name += self.ext
         with open(os.path.join(self.test_files, file_name), "r") as f:
             expected = csv.reader(f, delimiter=";")
             for row, col, value in expected:
@@ -42,18 +53,31 @@ class TestBase(unittest.TestCase):
                         value = float(value)
                     except ValueError:
                         pass
+                if replace is not None and isinstance(value, str):
+                    if replace in value:
+                        value = value.replace(replace, with_this)
                 self.assertEqual(sheet.cell(int(row), int(col)).value, value)
 
     def check_output_file(self, output, file_name):
-        file_name += ".test"
+        file_name += self.ext
         with open(os.path.join(self.test_files, file_name)) as f:
             expected = f.read()
         self.assertEqual(output.getvalue(), expected)
 
+    def replace_STree_version(self, line, output, index):
+        idx = line.find(self.stree_version)
+        return line.replace(self.stree_version, output[index][idx : idx + 5])
+
+    def replace_benchmark_version(self, line, output, index):
+        idx = line.find(self.benchmark_version)
+        return line.replace(
+            self.benchmark_version, output[index][idx : idx + 5]
+        )
+
     def check_file_file(self, computed_file, expected_file):
         with open(computed_file) as f:
             computed = f.read()
-        expected_file += ".test"
+        expected_file += self.ext
         with open(os.path.join(self.test_files, expected_file)) as f:
             expected = f.read()
         self.assertEqual(computed, expected)

benchmark/tests/Util_test.py

@@ -11,6 +11,8 @@ class UtilTest(TestBase):
         self.assertEqual("results", Folders.results)
         self.assertEqual("hidden_results", Folders.hidden_results)
         self.assertEqual("exreport", Folders.exreport)
+        self.assertEqual("excel", Folders.excel)
+        self.assertEqual("img", Folders.img)
         self.assertEqual(
             os.path.join(Folders.exreport, "exreport_output"), Folders.report
         )
@@ -116,7 +118,7 @@ class UtilTest(TestBase):
     def test_Files_get_results(self):
         os.chdir(os.path.dirname(os.path.abspath(__file__)))
         self.assertCountEqual(
-            Files().get_all_results(hidden=False),
+            Files.get_all_results(hidden=False),
             [
                 "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json",
                 "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
@@ -128,7 +130,7 @@ class UtilTest(TestBase):
             ],
         )
         self.assertCountEqual(
-            Files().get_all_results(hidden=True),
+            Files.get_all_results(hidden=True),
             [
                 "results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json",
                 "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_"
@@ -141,7 +143,7 @@ class UtilTest(TestBase):
         # check with results
         os.rename(Folders.results, f"{Folders.results}.test")
         try:
-            Files().get_all_results(hidden=False)
+            Files.get_all_results(hidden=False)
         except ValueError:
             pass
         else:
@@ -151,7 +153,7 @@ class UtilTest(TestBase):
         # check with hidden_results
         os.rename(Folders.hidden_results, f"{Folders.hidden_results}.test")
         try:
-            Files().get_all_results(hidden=True)
+            Files.get_all_results(hidden=True)
         except ValueError:
             pass
         else:
@@ -178,6 +180,13 @@ class UtilTest(TestBase):
             "model": "ODTE",
             "stratified": "0",
             "source_data": "Tanveer",
+            "seeds": "[57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]",
+            "discretize": "0",
+            "nodes": "Nodes",
+            "leaves": "Leaves",
+            "depth": "Depth",
+            "fit_features": "0",
+            "margin": "0.1",
         }
         computed = EnvData().load()
         self.assertDictEqual(computed, expected)

benchmark/tests/__init__.py

@@ -13,6 +13,7 @@ from .PairCheck_test import PairCheckTest
 from .Arguments_test import ArgumentsTest
 from .scripts.Be_Pair_check_test import BePairCheckTest
 from .scripts.Be_List_test import BeListTest
+from .scripts.Be_Init_Project_test import BeInitProjectTest
 from .scripts.Be_Report_test import BeReportTest
 from .scripts.Be_Summary_test import BeSummaryTest
 from .scripts.Be_Grid_test import BeGridTest

benchmark/tests/datasets/all.txt

@@ -1,2 +1,2 @@
-iris
-wine
+iris;class;all
+wine;class;[0, 1]

benchmark/tests/datasets/hayes-roth.arff

@@ -0,0 +1,305 @@
+% 1. Title: Hayes-Roth & Hayes-Roth (1977) Database
+% 
+% 2. Source Information:
+%    (a) Creators: Barbara and Frederick Hayes-Roth
+%    (b) Donor: David W. Aha (aha@ics.uci.edu) (714) 856-8779   
+%    (c) Date: March, 1989
+% 
+% 3. Past Usage:
+%     1. Hayes-Roth, B., & Hayes-Roth, F. (1977).  Concept learning and the
+%        recognition and classification of exemplars.  Journal of Verbal Learning
+%        and Verbal Behavior, 16, 321-338.
+%        -- Results: 
+%           -- Human subjects classification and recognition performance:
+% 	       1. decreases with distance from the prototype,
+% 	       2. is better on unseen prototypes than old instances, and
+% 	       3. improves with presentation frequency during learning.
+%     2. Anderson, J.R., & Kline, P.J. (1979).  A learning system and its 
+%        psychological implications.  In Proceedings of the Sixth International
+%        Joint Conference on Artificial Intelligence (pp. 16-21).  Tokyo, Japan:
+%        Morgan Kaufmann.
+%        -- Partitioned the results into 4 classes:
+% 	    1. prototypes
+% 	    2. near-prototypes with high presentation frequency during learning
+% 	    3. near-prototypes with low presentation frequency during learning
+% 	    4. instances that are far from protoypes
+%        -- Described evidence that ACT's classification confidence and
+%           recognition behaviors closely simulated human subjects' behaviors.
+%     3. Aha, D.W. (1989).  Incremental learning of independent, overlapping, and
+%        graded concept descriptions with an instance-based process framework.
+%        Manuscript submitted for publication.
+%        -- Used same partition as Anderson & Kline
+%        -- Described evidence that Bloom's classification confidence behavior
+% 	  is similar to the human subjects' behavior.  Bloom fitted the data
+% 	  more closely than did ACT. 
+% 
+% 4. Relevant Information:
+%      This database contains 5 numeric-valued attributes.  Only a subset of
+%      3 are used during testing (the latter 3).  Furthermore, only 2 of the
+%      3 concepts are "used" during testing (i.e., those with the prototypes
+%      000 and 111).  I've mapped all values to their zero-indexing equivalents.
+% 
+%      Some instances could be placed in either category 0 or 1.  I've followed
+%      the authors' suggestion, placing them in each category with equal
+%      probability.
+% 
+%      I've replaced the actual values of the attributes (i.e., hobby has values
+%      chess, sports and stamps) with numeric values.  I think this is how 
+%      the authors' did this when testing the categorization models described
+%      in the paper.  I find this unfair.  While the subjects were able to bring
+%      background knowledge to bear on the attribute values and their
+%      relationships, the algorithms were provided with no such knowledge.  I'm
+%      uncertain whether the 2 distractor attributes (name and hobby) are
+%      presented to the authors' algorithms during testing.  However, it is clear
+%      that only the age, educational status, and marital status attributes are
+%      given during the human subjects' transfer tests.  
+%     
+% 5. Number of Instances: 132 training instances, 28 test instances
+% 
+% 6. Number of Attributes: 5 plus the class membership attribute.  3 concepts.
+% 
+% 7. Attribute Information:
+%       -- 1. name: distinct for each instance and represented numerically
+%       -- 2. hobby: nominal values ranging between 1 and 3
+%       -- 3. age: nominal values ranging between 1 and 4
+%       -- 4. educational level: nominal values ranging between 1 and 4
+%       -- 5. marital status: nominal values ranging between 1 and 4
+%       -- 6. class: nominal value between 1 and 3
+% 
+% 9. Missing Attribute Values: none
+% 
+% 10. Class Distribution: see below
+% 
+% 11. Detailed description of the experiment:
+%   1. 3 categories (1, 2, and neither -- which I call 3)
+%      -- some of the instances could be classified in either class 1 or 2, and
+%         they have been evenly distributed between the two classes
+%   2. 5 Attributes
+%      -- A. name (a randomly-generated number between 1 and 132)
+%      -- B. hobby (a randomly-generated number between 1 and 3)
+%      -- C. age (a number between 1 and 4)
+%      -- D. education level (a number between 1 and 4)
+%      -- E. marital status (a number between 1 and 4)
+%   3. Classification: 
+%      -- only attributes C-E are diagnostic; values for A and B are ignored
+%      -- Class Neither: if a 4 occurs for any attribute C-E
+%      -- Class 1: Otherwise, if (# of 1's)>(# of 2's) for attributes C-E
+%      -- Class 2: Otherwise, if (# of 2's)>(# of 1's) for attributes C-E
+%      -- Either 1 or 2: Otherwise, if (# of 2's)=(# of 1's) for attributes C-E
+%   4. Prototypes:
+%      -- Class 1: 111
+%      -- Class 2: 222
+%      -- Class Either: 333
+%      -- Class Neither: 444  
+%   5. Number of training instances: 132
+%      -- Each instance presented 0, 1, or 10 times
+%      -- None of the prototypes seen during training
+%      -- 3 instances from each of categories 1, 2, and either are repeated 
+%         10 times each
+%      -- 3 additional instances from the Either category are shown during
+%         learning
+%   5. Number of test instances: 28
+%      -- All 9 class 1
+%      -- All 9 class 2
+%      -- All 6 class Either
+%      -- All 4 prototypes
+%      --------------------
+%      --    28 total
+% 
+% Observations of interest:
+%   1. Relative classification confidence of 
+%      -- prototypes for classes 1 and 2 (2 instances)
+%         (Anderson calls these Class 1 instances)
+%      -- instances of class 1 with frequency 10 during training and
+%         instances of class 2 with frequency 10 during training that
+%         are 1 value away from their respective prototypes (6 instances)
+%         (Anderson calls these Class 2 instances)
+%      -- instances of class 1 with frequency 1 during training and 
+%         instances of class 2 with frequency 1 during training that
+%         are 1 value away from their respective prototypes (6 instances)
+%         (Anderson calls these Class 3 instances)
+%      -- instances of class 1 with frequency 1 during training and 
+%         instances of class 2 with frequency 1 during training that
+%         are 2 values away from their respective prototypes (6 instances)
+%         (Anderson calls these Class 4 instances)
+%  2. Relative classification recognition of them also
+% 
+% Some Expected results:
+%    Both frequency and distance from prototype will effect the classification
+%    accuracy of instances.  Greater the frequency, higher the classification
+%    confidence.  Closer to prototype, higher the classification confidence.
+%
+% Information about the dataset
+% CLASSTYPE: nominal
+% CLASSINDEX: last
+%
+
+@relation hayes-roth
+
+@attribute hobby INTEGER
+@attribute age INTEGER
+@attribute educational_level INTEGER
+@attribute marital_status INTEGER
+@attribute class {1,2,3,4}
+
+@data
+2,1,1,2,1
+2,1,3,2,2
+3,1,4,1,3
+2,4,2,2,3
+1,1,3,4,3
+1,1,3,2,2
+3,1,3,2,2
+3,4,2,4,3
+2,2,1,1,1
+3,2,1,1,1
+1,2,1,1,1
+2,2,3,4,3
+1,1,2,1,1
+2,1,2,2,2
+2,4,1,4,3
+1,1,3,3,1
+3,2,1,2,2
+1,2,1,1,1
+3,3,2,1,1
+3,1,3,2,1
+1,2,2,1,2
+3,2,1,3,1
+2,1,2,1,1
+3,2,1,3,1
+2,3,2,1,1
+3,2,2,1,2
+3,2,1,3,2
+2,1,2,2,2
+1,1,3,2,1
+3,2,1,1,1
+1,4,1,1,3
+2,2,1,3,1
+1,2,1,3,2
+1,1,1,2,1
+2,4,3,1,3
+3,1,2,2,2
+1,1,2,2,2
+3,2,2,1,2
+1,2,1,2,2
+3,4,3,2,3
+2,2,2,1,2
+2,2,1,2,2
+3,2,1,3,2
+3,2,1,1,1
+3,1,2,1,1
+1,2,1,3,2
+2,1,1,2,1
+1,1,1,2,1
+1,2,2,3,2
+3,3,1,1,1
+3,3,3,1,1
+3,2,1,2,2
+3,2,1,2,2
+3,1,2,1,1
+1,1,1,2,1
+2,1,3,2,1
+2,2,2,1,2
+2,1,2,1,1
+2,2,1,3,1
+2,1,2,2,2
+1,2,4,2,3
+2,2,1,2,2
+1,1,2,4,3
+1,3,2,1,1
+2,4,4,2,3
+2,3,2,1,1
+3,1,2,2,2
+1,1,2,2,2
+1,3,2,4,3
+1,1,2,2,2
+3,1,4,2,3
+2,1,3,2,2
+1,1,3,2,2
+3,1,3,2,1
+1,2,4,4,3
+1,4,2,1,3
+2,1,2,1,1
+3,4,1,2,3
+2,2,1,1,1
+1,1,2,1,1
+2,2,4,3,3
+3,1,2,2,2
+1,1,3,2,1
+1,2,1,3,1
+1,4,4,1,3
+3,3,3,2,2
+2,2,1,3,2
+3,3,2,1,2
+1,1,1,3,1
+2,2,1,2,2
+2,2,2,1,2
+2,3,2,3,2
+1,3,2,1,2
+2,2,1,2,2
+1,1,1,2,1
+3,2,2,1,2
+3,2,1,1,1
+1,1,2,1,1
+3,1,4,4,3
+3,3,2,1,2
+2,3,2,1,2
+2,1,3,1,1
+1,2,1,2,2
+3,1,1,2,1
+2,2,4,1,3
+1,2,2,1,2
+2,3,2,1,2
+2,2,1,4,3
+1,4,2,3,3
+2,2,1,1,1
+1,2,1,1,1
+2,2,3,2,2
+1,3,2,1,1
+3,1,2,1,1
+3,1,1,2,1
+3,3,1,4,3
+2,3,4,1,3
+1,2,3,3,2
+3,3,2,2,2
+3,3,4,2,3
+1,2,2,1,2
+2,1,1,4,3
+3,1,2,2,2
+3,2,2,4,3
+2,3,1,3,1
+2,1,1,2,1
+3,4,1,3,3
+1,1,4,3,3
+2,1,2,1,1
+1,2,1,2,2
+1,2,2,1,2
+3,1,1,2,1
+1,1,1,2,1
+1,1,2,1,1
+1,2,1,1,1
+1,1,1,3,1
+1,1,3,1,1
+1,3,1,1,1
+1,1,3,3,1
+1,3,1,3,1
+1,3,3,1,1
+1,2,2,1,2
+1,2,1,2,2
+1,1,2,2,2
+1,2,2,3,2
+1,2,3,2,2
+1,3,2,2,2
+1,2,3,3,2
+1,3,2,3,2
+1,3,3,2,2
+1,1,3,2,1
+1,3,2,1,2
+1,2,1,3,1
+1,2,3,1,2
+1,1,2,3,1
+1,3,1,2,2
+1,1,1,1,1
+1,2,2,2,2
+1,3,3,3,1
+1,4,4,4,3

benchmark/tests/datasets/iris.arff

@@ -0,0 +1,225 @@
+% 1. Title: Iris Plants Database
+% 
+% 2. Sources:
+%      (a) Creator: R.A. Fisher
+%      (b) Donor: Michael Marshall (MARSHALL%PLU@io.arc.nasa.gov)
+%      (c) Date: July, 1988
+% 
+% 3. Past Usage:
+%    - Publications: too many to mention!!!  Here are a few.
+%    1. Fisher,R.A. "The use of multiple measurements in taxonomic problems"
+%       Annual Eugenics, 7, Part II, 179-188 (1936); also in "Contributions
+%       to Mathematical Statistics" (John Wiley, NY, 1950).
+%    2. Duda,R.O., & Hart,P.E. (1973) Pattern Classification and Scene Analysis.
+%       (Q327.D83) John Wiley & Sons.  ISBN 0-471-22361-1.  See page 218.
+%    3. Dasarathy, B.V. (1980) "Nosing Around the Neighborhood: A New System
+%       Structure and Classification Rule for Recognition in Partially Exposed
+%       Environments".  IEEE Transactions on Pattern Analysis and Machine
+%       Intelligence, Vol. PAMI-2, No. 1, 67-71.
+%       -- Results:
+%          -- very low misclassification rates (0% for the setosa class)
+%    4. Gates, G.W. (1972) "The Reduced Nearest Neighbor Rule".  IEEE 
+%       Transactions on Information Theory, May 1972, 431-433.
+%       -- Results:
+%          -- very low misclassification rates again
+%    5. See also: 1988 MLC Proceedings, 54-64.  Cheeseman et al's AUTOCLASS II
+%       conceptual clustering system finds 3 classes in the data.
+% 
+% 4. Relevant Information:
+%    --- This is perhaps the best known database to be found in the pattern
+%        recognition literature.  Fisher's paper is a classic in the field
+%        and is referenced frequently to this day.  (See Duda & Hart, for
+%        example.)  The data set contains 3 classes of 50 instances each,
+%        where each class refers to a type of iris plant.  One class is
+%        linearly separable from the other 2; the latter are NOT linearly
+%        separable from each other.
+%    --- Predicted attribute: class of iris plant.
+%    --- This is an exceedingly simple domain.
+% 
+% 5. Number of Instances: 150 (50 in each of three classes)
+% 
+% 6. Number of Attributes: 4 numeric, predictive attributes and the class
+% 
+% 7. Attribute Information:
+%    1. sepal length in cm
+%    2. sepal width in cm
+%    3. petal length in cm
+%    4. petal width in cm
+%    5. class: 
+%       -- Iris Setosa
+%       -- Iris Versicolour
+%       -- Iris Virginica
+% 
+% 8. Missing Attribute Values: None
+% 
+% Summary Statistics:
+%  	           Min  Max   Mean    SD   Class Correlation
+%    sepal length: 4.3  7.9   5.84  0.83    0.7826   
+%     sepal width: 2.0  4.4   3.05  0.43   -0.4194
+%    petal length: 1.0  6.9   3.76  1.76    0.9490  (high!)
+%     petal width: 0.1  2.5   1.20  0.76    0.9565  (high!)
+% 
+% 9. Class Distribution: 33.3% for each of 3 classes.
+
+@RELATION iris
+
+@ATTRIBUTE sepallength	REAL
+@ATTRIBUTE sepalwidth 	REAL
+@ATTRIBUTE petallength 	REAL
+@ATTRIBUTE petalwidth	REAL
+@ATTRIBUTE class 	{Iris-setosa,Iris-versicolor,Iris-virginica}
+
+@DATA
+5.1,3.5,1.4,0.2,Iris-setosa
+4.9,3.0,1.4,0.2,Iris-setosa
+4.7,3.2,1.3,0.2,Iris-setosa
+4.6,3.1,1.5,0.2,Iris-setosa
+5.0,3.6,1.4,0.2,Iris-setosa
+5.4,3.9,1.7,0.4,Iris-setosa
+4.6,3.4,1.4,0.3,Iris-setosa
+5.0,3.4,1.5,0.2,Iris-setosa
+4.4,2.9,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.4,3.7,1.5,0.2,Iris-setosa
+4.8,3.4,1.6,0.2,Iris-setosa
+4.8,3.0,1.4,0.1,Iris-setosa
+4.3,3.0,1.1,0.1,Iris-setosa
+5.8,4.0,1.2,0.2,Iris-setosa
+5.7,4.4,1.5,0.4,Iris-setosa
+5.4,3.9,1.3,0.4,Iris-setosa
+5.1,3.5,1.4,0.3,Iris-setosa
+5.7,3.8,1.7,0.3,Iris-setosa
+5.1,3.8,1.5,0.3,Iris-setosa
+5.4,3.4,1.7,0.2,Iris-setosa
+5.1,3.7,1.5,0.4,Iris-setosa
+4.6,3.6,1.0,0.2,Iris-setosa
+5.1,3.3,1.7,0.5,Iris-setosa
+4.8,3.4,1.9,0.2,Iris-setosa
+5.0,3.0,1.6,0.2,Iris-setosa
+5.0,3.4,1.6,0.4,Iris-setosa
+5.2,3.5,1.5,0.2,Iris-setosa
+5.2,3.4,1.4,0.2,Iris-setosa
+4.7,3.2,1.6,0.2,Iris-setosa
+4.8,3.1,1.6,0.2,Iris-setosa
+5.4,3.4,1.5,0.4,Iris-setosa
+5.2,4.1,1.5,0.1,Iris-setosa
+5.5,4.2,1.4,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+5.0,3.2,1.2,0.2,Iris-setosa
+5.5,3.5,1.3,0.2,Iris-setosa
+4.9,3.1,1.5,0.1,Iris-setosa
+4.4,3.0,1.3,0.2,Iris-setosa
+5.1,3.4,1.5,0.2,Iris-setosa
+5.0,3.5,1.3,0.3,Iris-setosa
+4.5,2.3,1.3,0.3,Iris-setosa
+4.4,3.2,1.3,0.2,Iris-setosa
+5.0,3.5,1.6,0.6,Iris-setosa
+5.1,3.8,1.9,0.4,Iris-setosa
+4.8,3.0,1.4,0.3,Iris-setosa
+5.1,3.8,1.6,0.2,Iris-setosa
+4.6,3.2,1.4,0.2,Iris-setosa
+5.3,3.7,1.5,0.2,Iris-setosa
+5.0,3.3,1.4,0.2,Iris-setosa
+7.0,3.2,4.7,1.4,Iris-versicolor
+6.4,3.2,4.5,1.5,Iris-versicolor
+6.9,3.1,4.9,1.5,Iris-versicolor
+5.5,2.3,4.0,1.3,Iris-versicolor
+6.5,2.8,4.6,1.5,Iris-versicolor
+5.7,2.8,4.5,1.3,Iris-versicolor
+6.3,3.3,4.7,1.6,Iris-versicolor
+4.9,2.4,3.3,1.0,Iris-versicolor
+6.6,2.9,4.6,1.3,Iris-versicolor
+5.2,2.7,3.9,1.4,Iris-versicolor
+5.0,2.0,3.5,1.0,Iris-versicolor
+5.9,3.0,4.2,1.5,Iris-versicolor
+6.0,2.2,4.0,1.0,Iris-versicolor
+6.1,2.9,4.7,1.4,Iris-versicolor
+5.6,2.9,3.6,1.3,Iris-versicolor
+6.7,3.1,4.4,1.4,Iris-versicolor
+5.6,3.0,4.5,1.5,Iris-versicolor
+5.8,2.7,4.1,1.0,Iris-versicolor
+6.2,2.2,4.5,1.5,Iris-versicolor
+5.6,2.5,3.9,1.1,Iris-versicolor
+5.9,3.2,4.8,1.8,Iris-versicolor
+6.1,2.8,4.0,1.3,Iris-versicolor
+6.3,2.5,4.9,1.5,Iris-versicolor
+6.1,2.8,4.7,1.2,Iris-versicolor
+6.4,2.9,4.3,1.3,Iris-versicolor
+6.6,3.0,4.4,1.4,Iris-versicolor
+6.8,2.8,4.8,1.4,Iris-versicolor
+6.7,3.0,5.0,1.7,Iris-versicolor
+6.0,2.9,4.5,1.5,Iris-versicolor
+5.7,2.6,3.5,1.0,Iris-versicolor
+5.5,2.4,3.8,1.1,Iris-versicolor
+5.5,2.4,3.7,1.0,Iris-versicolor
+5.8,2.7,3.9,1.2,Iris-versicolor
+6.0,2.7,5.1,1.6,Iris-versicolor
+5.4,3.0,4.5,1.5,Iris-versicolor
+6.0,3.4,4.5,1.6,Iris-versicolor
+6.7,3.1,4.7,1.5,Iris-versicolor
+6.3,2.3,4.4,1.3,Iris-versicolor
+5.6,3.0,4.1,1.3,Iris-versicolor
+5.5,2.5,4.0,1.3,Iris-versicolor
+5.5,2.6,4.4,1.2,Iris-versicolor
+6.1,3.0,4.6,1.4,Iris-versicolor
+5.8,2.6,4.0,1.2,Iris-versicolor
+5.0,2.3,3.3,1.0,Iris-versicolor
+5.6,2.7,4.2,1.3,Iris-versicolor
+5.7,3.0,4.2,1.2,Iris-versicolor
+5.7,2.9,4.2,1.3,Iris-versicolor
+6.2,2.9,4.3,1.3,Iris-versicolor
+5.1,2.5,3.0,1.1,Iris-versicolor
+5.7,2.8,4.1,1.3,Iris-versicolor
+6.3,3.3,6.0,2.5,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+7.1,3.0,5.9,2.1,Iris-virginica
+6.3,2.9,5.6,1.8,Iris-virginica
+6.5,3.0,5.8,2.2,Iris-virginica
+7.6,3.0,6.6,2.1,Iris-virginica
+4.9,2.5,4.5,1.7,Iris-virginica
+7.3,2.9,6.3,1.8,Iris-virginica
+6.7,2.5,5.8,1.8,Iris-virginica
+7.2,3.6,6.1,2.5,Iris-virginica
+6.5,3.2,5.1,2.0,Iris-virginica
+6.4,2.7,5.3,1.9,Iris-virginica
+6.8,3.0,5.5,2.1,Iris-virginica
+5.7,2.5,5.0,2.0,Iris-virginica
+5.8,2.8,5.1,2.4,Iris-virginica
+6.4,3.2,5.3,2.3,Iris-virginica
+6.5,3.0,5.5,1.8,Iris-virginica
+7.7,3.8,6.7,2.2,Iris-virginica
+7.7,2.6,6.9,2.3,Iris-virginica
+6.0,2.2,5.0,1.5,Iris-virginica
+6.9,3.2,5.7,2.3,Iris-virginica
+5.6,2.8,4.9,2.0,Iris-virginica
+7.7,2.8,6.7,2.0,Iris-virginica
+6.3,2.7,4.9,1.8,Iris-virginica
+6.7,3.3,5.7,2.1,Iris-virginica
+7.2,3.2,6.0,1.8,Iris-virginica
+6.2,2.8,4.8,1.8,Iris-virginica
+6.1,3.0,4.9,1.8,Iris-virginica
+6.4,2.8,5.6,2.1,Iris-virginica
+7.2,3.0,5.8,1.6,Iris-virginica
+7.4,2.8,6.1,1.9,Iris-virginica
+7.9,3.8,6.4,2.0,Iris-virginica
+6.4,2.8,5.6,2.2,Iris-virginica
+6.3,2.8,5.1,1.5,Iris-virginica
+6.1,2.6,5.6,1.4,Iris-virginica
+7.7,3.0,6.1,2.3,Iris-virginica
+6.3,3.4,5.6,2.4,Iris-virginica
+6.4,3.1,5.5,1.8,Iris-virginica
+6.0,3.0,4.8,1.8,Iris-virginica
+6.9,3.1,5.4,2.1,Iris-virginica
+6.7,3.1,5.6,2.4,Iris-virginica
+6.9,3.1,5.1,2.3,Iris-virginica
+5.8,2.7,5.1,1.9,Iris-virginica
+6.8,3.2,5.9,2.3,Iris-virginica
+6.7,3.3,5.7,2.5,Iris-virginica
+6.7,3.0,5.2,2.3,Iris-virginica
+6.3,2.5,5.0,1.9,Iris-virginica
+6.5,3.0,5.2,2.0,Iris-virginica
+6.2,3.4,5.4,2.3,Iris-virginica
+5.9,3.0,5.1,1.8,Iris-virginica
+%
+%
+%

benchmark/tests/datasets/wine.arff

@@ -0,0 +1,302 @@
+% 1. Title of Database: Wine recognition data
+% 	Updated Sept 21, 1998 by C.Blake : Added attribute information
+% 
+% 2. Sources:
+%    (a) Forina, M. et al, PARVUS - An Extendible Package for Data
+%        Exploration, Classification and Correlation. Institute of Pharmaceutical
+%        and Food Analysis and Technologies, Via Brigata Salerno, 
+%        16147 Genoa, Italy.
+% 
+%    (b) Stefan Aeberhard, email: stefan@coral.cs.jcu.edu.au
+%    (c) July 1991
+% 3. Past Usage:
+% 
+%    (1)
+%    S. Aeberhard, D. Coomans and O. de Vel,
+%    Comparison of Classifiers in High Dimensional Settings,
+%    Tech. Rep. no. 92-02, (1992), Dept. of Computer Science and Dept. of
+%    Mathematics and Statistics, James Cook University of North Queensland.
+%    (Also submitted to Technometrics).
+% 
+%    The data was used with many others for comparing various 
+%    classifiers. The classes are separable, though only RDA 
+%    has achieved 100% correct classification.
+%    (RDA : 100%, QDA 99.4%, LDA 98.9%, 1NN 96.1% (z-transformed data))
+%    (All results using the leave-one-out technique)
+% 
+%    In a classification context, this is a well posed problem 
+%    with "well behaved" class structures. A good data set 
+%    for first testing of a new classifier, but not very 
+%    challenging.
+% 
+%    (2) 
+%    S. Aeberhard, D. Coomans and O. de Vel,
+%    "THE CLASSIFICATION PERFORMANCE OF RDA"
+%    Tech. Rep. no. 92-01, (1992), Dept. of Computer Science and Dept. of
+%    Mathematics and Statistics, James Cook University of North Queensland.
+%    (Also submitted to Journal of Chemometrics).
+% 
+%    Here, the data was used to illustrate the superior performance of
+%    the use of a new appreciation function with RDA. 
+% 
+% 4. Relevant Information:
+% 
+%    -- These data are the results of a chemical analysis of
+%       wines grown in the same region in Italy but derived from three
+%       different cultivars.
+%       The analysis determined the quantities of 13 constituents
+%       found in each of the three types of wines. 
+% 
+%    -- I think that the initial data set had around 30 variables, but 
+%       for some reason I only have the 13 dimensional version. 
+%       I had a list of what the 30 or so variables were, but a.) 
+%       I lost it, and b.), I would not know which 13 variables
+%       are included in the set.
+% 
+%    -- The attributes are (dontated by Riccardo Leardi, 
+% 	riclea@anchem.unige.it )
+%  	1) Alcohol
+%  	2) Malic acid
+%  	3) Ash
+% 	4) Alcalinity of ash  
+%  	5) Magnesium
+% 	6) Total phenols
+%  	7) Flavanoids
+%  	8) Nonflavanoid phenols
+%  	9) Proanthocyanins
+% 	10)Color intensity
+%  	11)Hue
+%  	12)OD280/OD315 of diluted wines
+%  	13)Proline            
+% 
+% 5. Number of Instances
+% 
+%       	class 1 59
+% 	class 2 71
+% 	class 3 48
+% 
+% 6. Number of Attributes 
+% 	
+% 	13
+% 
+% 7. For Each Attribute:
+% 
+% 	All attributes are continuous
+% 	
+% 	No statistics available, but suggest to standardise
+% 	variables for certain uses (e.g. for us with classifiers
+% 	which are NOT scale invariant)
+% 
+% 	NOTE: 1st attribute is class identifier (1-3)
+% 
+% 8. Missing Attribute Values:
+% 
+% 	None
+% 
+% 9. Class Distribution: number of instances per class
+% 
+%       	class 1 59
+% 	class 2 71
+% 	class 3 48
+%
+% Information about the dataset
+% CLASSTYPE: nominal
+% CLASSINDEX: first
+%
+
+@relation wine
+
+@attribute class {1,2,3}
+@attribute Alcohol REAL
+@attribute Malic_acid REAL
+@attribute Ash REAL
+@attribute Alcalinity_of_ash REAL
+@attribute Magnesium INTEGER
+@attribute Total_phenols REAL
+@attribute Flavanoids REAL
+@attribute Nonflavanoid_phenols REAL
+@attribute Proanthocyanins REAL
+@attribute Color_intensity REAL
+@attribute Hue REAL
+@attribute OD280/OD315_of_diluted_wines REAL
+@attribute Proline INTEGER
+
+@data
+1,14.23,1.71,2.43,15.6,127,2.8,3.06,.28,2.29,5.64,1.04,3.92,1065
+1,13.2,1.78,2.14,11.2,100,2.65,2.76,.26,1.28,4.38,1.05,3.4,1050
+1,13.16,2.36,2.67,18.6,101,2.8,3.24,.3,2.81,5.68,1.03,3.17,1185
+1,14.37,1.95,2.5,16.8,113,3.85,3.49,.24,2.18,7.8,.86,3.45,1480
+1,13.24,2.59,2.87,21,118,2.8,2.69,.39,1.82,4.32,1.04,2.93,735
+1,14.2,1.76,2.45,15.2,112,3.27,3.39,.34,1.97,6.75,1.05,2.85,1450
+1,14.39,1.87,2.45,14.6,96,2.5,2.52,.3,1.98,5.25,1.02,3.58,1290
+1,14.06,2.15,2.61,17.6,121,2.6,2.51,.31,1.25,5.05,1.06,3.58,1295
+1,14.83,1.64,2.17,14,97,2.8,2.98,.29,1.98,5.2,1.08,2.85,1045
+1,13.86,1.35,2.27,16,98,2.98,3.15,.22,1.85,7.22,1.01,3.55,1045
+1,14.1,2.16,2.3,18,105,2.95,3.32,.22,2.38,5.75,1.25,3.17,1510
+1,14.12,1.48,2.32,16.8,95,2.2,2.43,.26,1.57,5,1.17,2.82,1280
+1,13.75,1.73,2.41,16,89,2.6,2.76,.29,1.81,5.6,1.15,2.9,1320
+1,14.75,1.73,2.39,11.4,91,3.1,3.69,.43,2.81,5.4,1.25,2.73,1150
+1,14.38,1.87,2.38,12,102,3.3,3.64,.29,2.96,7.5,1.2,3,1547
+1,13.63,1.81,2.7,17.2,112,2.85,2.91,.3,1.46,7.3,1.28,2.88,1310
+1,14.3,1.92,2.72,20,120,2.8,3.14,.33,1.97,6.2,1.07,2.65,1280
+1,13.83,1.57,2.62,20,115,2.95,3.4,.4,1.72,6.6,1.13,2.57,1130
+1,14.19,1.59,2.48,16.5,108,3.3,3.93,.32,1.86,8.7,1.23,2.82,1680
+1,13.64,3.1,2.56,15.2,116,2.7,3.03,.17,1.66,5.1,.96,3.36,845
+1,14.06,1.63,2.28,16,126,3,3.17,.24,2.1,5.65,1.09,3.71,780
+1,12.93,3.8,2.65,18.6,102,2.41,2.41,.25,1.98,4.5,1.03,3.52,770
+1,13.71,1.86,2.36,16.6,101,2.61,2.88,.27,1.69,3.8,1.11,4,1035
+1,12.85,1.6,2.52,17.8,95,2.48,2.37,.26,1.46,3.93,1.09,3.63,1015
+1,13.5,1.81,2.61,20,96,2.53,2.61,.28,1.66,3.52,1.12,3.82,845
+1,13.05,2.05,3.22,25,124,2.63,2.68,.47,1.92,3.58,1.13,3.2,830
+1,13.39,1.77,2.62,16.1,93,2.85,2.94,.34,1.45,4.8,.92,3.22,1195
+1,13.3,1.72,2.14,17,94,2.4,2.19,.27,1.35,3.95,1.02,2.77,1285
+1,13.87,1.9,2.8,19.4,107,2.95,2.97,.37,1.76,4.5,1.25,3.4,915
+1,14.02,1.68,2.21,16,96,2.65,2.33,.26,1.98,4.7,1.04,3.59,1035
+1,13.73,1.5,2.7,22.5,101,3,3.25,.29,2.38,5.7,1.19,2.71,1285
+1,13.58,1.66,2.36,19.1,106,2.86,3.19,.22,1.95,6.9,1.09,2.88,1515
+1,13.68,1.83,2.36,17.2,104,2.42,2.69,.42,1.97,3.84,1.23,2.87,990
+1,13.76,1.53,2.7,19.5,132,2.95,2.74,.5,1.35,5.4,1.25,3,1235
+1,13.51,1.8,2.65,19,110,2.35,2.53,.29,1.54,4.2,1.1,2.87,1095
+1,13.48,1.81,2.41,20.5,100,2.7,2.98,.26,1.86,5.1,1.04,3.47,920
+1,13.28,1.64,2.84,15.5,110,2.6,2.68,.34,1.36,4.6,1.09,2.78,880
+1,13.05,1.65,2.55,18,98,2.45,2.43,.29,1.44,4.25,1.12,2.51,1105
+1,13.07,1.5,2.1,15.5,98,2.4,2.64,.28,1.37,3.7,1.18,2.69,1020
+1,14.22,3.99,2.51,13.2,128,3,3.04,.2,2.08,5.1,.89,3.53,760
+1,13.56,1.71,2.31,16.2,117,3.15,3.29,.34,2.34,6.13,.95,3.38,795
+1,13.41,3.84,2.12,18.8,90,2.45,2.68,.27,1.48,4.28,.91,3,1035
+1,13.88,1.89,2.59,15,101,3.25,3.56,.17,1.7,5.43,.88,3.56,1095
+1,13.24,3.98,2.29,17.5,103,2.64,2.63,.32,1.66,4.36,.82,3,680
+1,13.05,1.77,2.1,17,107,3,3,.28,2.03,5.04,.88,3.35,885
+1,14.21,4.04,2.44,18.9,111,2.85,2.65,.3,1.25,5.24,.87,3.33,1080
+1,14.38,3.59,2.28,16,102,3.25,3.17,.27,2.19,4.9,1.04,3.44,1065
+1,13.9,1.68,2.12,16,101,3.1,3.39,.21,2.14,6.1,.91,3.33,985
+1,14.1,2.02,2.4,18.8,103,2.75,2.92,.32,2.38,6.2,1.07,2.75,1060
+1,13.94,1.73,2.27,17.4,108,2.88,3.54,.32,2.08,8.90,1.12,3.1,1260
+1,13.05,1.73,2.04,12.4,92,2.72,3.27,.17,2.91,7.2,1.12,2.91,1150
+1,13.83,1.65,2.6,17.2,94,2.45,2.99,.22,2.29,5.6,1.24,3.37,1265
+1,13.82,1.75,2.42,14,111,3.88,3.74,.32,1.87,7.05,1.01,3.26,1190
+1,13.77,1.9,2.68,17.1,115,3,2.79,.39,1.68,6.3,1.13,2.93,1375
+1,13.74,1.67,2.25,16.4,118,2.6,2.9,.21,1.62,5.85,.92,3.2,1060
+1,13.56,1.73,2.46,20.5,116,2.96,2.78,.2,2.45,6.25,.98,3.03,1120
+1,14.22,1.7,2.3,16.3,118,3.2,3,.26,2.03,6.38,.94,3.31,970
+1,13.29,1.97,2.68,16.8,102,3,3.23,.31,1.66,6,1.07,2.84,1270
+1,13.72,1.43,2.5,16.7,108,3.4,3.67,.19,2.04,6.8,.89,2.87,1285
+2,12.37,.94,1.36,10.6,88,1.98,.57,.28,.42,1.95,1.05,1.82,520
+2,12.33,1.1,2.28,16,101,2.05,1.09,.63,.41,3.27,1.25,1.67,680
+2,12.64,1.36,2.02,16.8,100,2.02,1.41,.53,.62,5.75,.98,1.59,450
+2,13.67,1.25,1.92,18,94,2.1,1.79,.32,.73,3.8,1.23,2.46,630
+2,12.37,1.13,2.16,19,87,3.5,3.1,.19,1.87,4.45,1.22,2.87,420
+2,12.17,1.45,2.53,19,104,1.89,1.75,.45,1.03,2.95,1.45,2.23,355
+2,12.37,1.21,2.56,18.1,98,2.42,2.65,.37,2.08,4.6,1.19,2.3,678
+2,13.11,1.01,1.7,15,78,2.98,3.18,.26,2.28,5.3,1.12,3.18,502
+2,12.37,1.17,1.92,19.6,78,2.11,2,.27,1.04,4.68,1.12,3.48,510
+2,13.34,.94,2.36,17,110,2.53,1.3,.55,.42,3.17,1.02,1.93,750
+2,12.21,1.19,1.75,16.8,151,1.85,1.28,.14,2.5,2.85,1.28,3.07,718
+2,12.29,1.61,2.21,20.4,103,1.1,1.02,.37,1.46,3.05,.906,1.82,870
+2,13.86,1.51,2.67,25,86,2.95,2.86,.21,1.87,3.38,1.36,3.16,410
+2,13.49,1.66,2.24,24,87,1.88,1.84,.27,1.03,3.74,.98,2.78,472
+2,12.99,1.67,2.6,30,139,3.3,2.89,.21,1.96,3.35,1.31,3.5,985
+2,11.96,1.09,2.3,21,101,3.38,2.14,.13,1.65,3.21,.99,3.13,886
+2,11.66,1.88,1.92,16,97,1.61,1.57,.34,1.15,3.8,1.23,2.14,428
+2,13.03,.9,1.71,16,86,1.95,2.03,.24,1.46,4.6,1.19,2.48,392
+2,11.84,2.89,2.23,18,112,1.72,1.32,.43,.95,2.65,.96,2.52,500
+2,12.33,.99,1.95,14.8,136,1.9,1.85,.35,2.76,3.4,1.06,2.31,750
+2,12.7,3.87,2.4,23,101,2.83,2.55,.43,1.95,2.57,1.19,3.13,463
+2,12,.92,2,19,86,2.42,2.26,.3,1.43,2.5,1.38,3.12,278
+2,12.72,1.81,2.2,18.8,86,2.2,2.53,.26,1.77,3.9,1.16,3.14,714
+2,12.08,1.13,2.51,24,78,2,1.58,.4,1.4,2.2,1.31,2.72,630
+2,13.05,3.86,2.32,22.5,85,1.65,1.59,.61,1.62,4.8,.84,2.01,515
+2,11.84,.89,2.58,18,94,2.2,2.21,.22,2.35,3.05,.79,3.08,520
+2,12.67,.98,2.24,18,99,2.2,1.94,.3,1.46,2.62,1.23,3.16,450
+2,12.16,1.61,2.31,22.8,90,1.78,1.69,.43,1.56,2.45,1.33,2.26,495
+2,11.65,1.67,2.62,26,88,1.92,1.61,.4,1.34,2.6,1.36,3.21,562
+2,11.64,2.06,2.46,21.6,84,1.95,1.69,.48,1.35,2.8,1,2.75,680
+2,12.08,1.33,2.3,23.6,70,2.2,1.59,.42,1.38,1.74,1.07,3.21,625
+2,12.08,1.83,2.32,18.5,81,1.6,1.5,.52,1.64,2.4,1.08,2.27,480
+2,12,1.51,2.42,22,86,1.45,1.25,.5,1.63,3.6,1.05,2.65,450
+2,12.69,1.53,2.26,20.7,80,1.38,1.46,.58,1.62,3.05,.96,2.06,495
+2,12.29,2.83,2.22,18,88,2.45,2.25,.25,1.99,2.15,1.15,3.3,290
+2,11.62,1.99,2.28,18,98,3.02,2.26,.17,1.35,3.25,1.16,2.96,345
+2,12.47,1.52,2.2,19,162,2.5,2.27,.32,3.28,2.6,1.16,2.63,937
+2,11.81,2.12,2.74,21.5,134,1.6,.99,.14,1.56,2.5,.95,2.26,625
+2,12.29,1.41,1.98,16,85,2.55,2.5,.29,1.77,2.9,1.23,2.74,428
+2,12.37,1.07,2.1,18.5,88,3.52,3.75,.24,1.95,4.5,1.04,2.77,660
+2,12.29,3.17,2.21,18,88,2.85,2.99,.45,2.81,2.3,1.42,2.83,406
+2,12.08,2.08,1.7,17.5,97,2.23,2.17,.26,1.4,3.3,1.27,2.96,710
+2,12.6,1.34,1.9,18.5,88,1.45,1.36,.29,1.35,2.45,1.04,2.77,562
+2,12.34,2.45,2.46,21,98,2.56,2.11,.34,1.31,2.8,.8,3.38,438
+2,11.82,1.72,1.88,19.5,86,2.5,1.64,.37,1.42,2.06,.94,2.44,415
+2,12.51,1.73,1.98,20.5,85,2.2,1.92,.32,1.48,2.94,1.04,3.57,672
+2,12.42,2.55,2.27,22,90,1.68,1.84,.66,1.42,2.7,.86,3.3,315
+2,12.25,1.73,2.12,19,80,1.65,2.03,.37,1.63,3.4,1,3.17,510
+2,12.72,1.75,2.28,22.5,84,1.38,1.76,.48,1.63,3.3,.88,2.42,488
+2,12.22,1.29,1.94,19,92,2.36,2.04,.39,2.08,2.7,.86,3.02,312
+2,11.61,1.35,2.7,20,94,2.74,2.92,.29,2.49,2.65,.96,3.26,680
+2,11.46,3.74,1.82,19.5,107,3.18,2.58,.24,3.58,2.9,.75,2.81,562
+2,12.52,2.43,2.17,21,88,2.55,2.27,.26,1.22,2,.9,2.78,325
+2,11.76,2.68,2.92,20,103,1.75,2.03,.6,1.05,3.8,1.23,2.5,607
+2,11.41,.74,2.5,21,88,2.48,2.01,.42,1.44,3.08,1.1,2.31,434
+2,12.08,1.39,2.5,22.5,84,2.56,2.29,.43,1.04,2.9,.93,3.19,385
+2,11.03,1.51,2.2,21.5,85,2.46,2.17,.52,2.01,1.9,1.71,2.87,407
+2,11.82,1.47,1.99,20.8,86,1.98,1.6,.3,1.53,1.95,.95,3.33,495
+2,12.42,1.61,2.19,22.5,108,2,2.09,.34,1.61,2.06,1.06,2.96,345
+2,12.77,3.43,1.98,16,80,1.63,1.25,.43,.83,3.4,.7,2.12,372
+2,12,3.43,2,19,87,2,1.64,.37,1.87,1.28,.93,3.05,564
+2,11.45,2.4,2.42,20,96,2.9,2.79,.32,1.83,3.25,.8,3.39,625
+2,11.56,2.05,3.23,28.5,119,3.18,5.08,.47,1.87,6,.93,3.69,465
+2,12.42,4.43,2.73,26.5,102,2.2,2.13,.43,1.71,2.08,.92,3.12,365
+2,13.05,5.8,2.13,21.5,86,2.62,2.65,.3,2.01,2.6,.73,3.1,380
+2,11.87,4.31,2.39,21,82,2.86,3.03,.21,2.91,2.8,.75,3.64,380
+2,12.07,2.16,2.17,21,85,2.6,2.65,.37,1.35,2.76,.86,3.28,378
+2,12.43,1.53,2.29,21.5,86,2.74,3.15,.39,1.77,3.94,.69,2.84,352
+2,11.79,2.13,2.78,28.5,92,2.13,2.24,.58,1.76,3,.97,2.44,466
+2,12.37,1.63,2.3,24.5,88,2.22,2.45,.4,1.9,2.12,.89,2.78,342
+2,12.04,4.3,2.38,22,80,2.1,1.75,.42,1.35,2.6,.79,2.57,580
+3,12.86,1.35,2.32,18,122,1.51,1.25,.21,.94,4.1,.76,1.29,630
+3,12.88,2.99,2.4,20,104,1.3,1.22,.24,.83,5.4,.74,1.42,530
+3,12.81,2.31,2.4,24,98,1.15,1.09,.27,.83,5.7,.66,1.36,560
+3,12.7,3.55,2.36,21.5,106,1.7,1.2,.17,.84,5,.78,1.29,600
+3,12.51,1.24,2.25,17.5,85,2,.58,.6,1.25,5.45,.75,1.51,650
+3,12.6,2.46,2.2,18.5,94,1.62,.66,.63,.94,7.1,.73,1.58,695
+3,12.25,4.72,2.54,21,89,1.38,.47,.53,.8,3.85,.75,1.27,720
+3,12.53,5.51,2.64,25,96,1.79,.6,.63,1.1,5,.82,1.69,515
+3,13.49,3.59,2.19,19.5,88,1.62,.48,.58,.88,5.7,.81,1.82,580
+3,12.84,2.96,2.61,24,101,2.32,.6,.53,.81,4.92,.89,2.15,590
+3,12.93,2.81,2.7,21,96,1.54,.5,.53,.75,4.6,.77,2.31,600
+3,13.36,2.56,2.35,20,89,1.4,.5,.37,.64,5.6,.7,2.47,780
+3,13.52,3.17,2.72,23.5,97,1.55,.52,.5,.55,4.35,.89,2.06,520
+3,13.62,4.95,2.35,20,92,2,.8,.47,1.02,4.4,.91,2.05,550
+3,12.25,3.88,2.2,18.5,112,1.38,.78,.29,1.14,8.21,.65,2,855
+3,13.16,3.57,2.15,21,102,1.5,.55,.43,1.3,4,.6,1.68,830
+3,13.88,5.04,2.23,20,80,.98,.34,.4,.68,4.9,.58,1.33,415
+3,12.87,4.61,2.48,21.5,86,1.7,.65,.47,.86,7.65,.54,1.86,625
+3,13.32,3.24,2.38,21.5,92,1.93,.76,.45,1.25,8.42,.55,1.62,650
+3,13.08,3.9,2.36,21.5,113,1.41,1.39,.34,1.14,9.40,.57,1.33,550
+3,13.5,3.12,2.62,24,123,1.4,1.57,.22,1.25,8.60,.59,1.3,500
+3,12.79,2.67,2.48,22,112,1.48,1.36,.24,1.26,10.8,.48,1.47,480
+3,13.11,1.9,2.75,25.5,116,2.2,1.28,.26,1.56,7.1,.61,1.33,425
+3,13.23,3.3,2.28,18.5,98,1.8,.83,.61,1.87,10.52,.56,1.51,675
+3,12.58,1.29,2.1,20,103,1.48,.58,.53,1.4,7.6,.58,1.55,640
+3,13.17,5.19,2.32,22,93,1.74,.63,.61,1.55,7.9,.6,1.48,725
+3,13.84,4.12,2.38,19.5,89,1.8,.83,.48,1.56,9.01,.57,1.64,480
+3,12.45,3.03,2.64,27,97,1.9,.58,.63,1.14,7.5,.67,1.73,880
+3,14.34,1.68,2.7,25,98,2.8,1.31,.53,2.7,13,.57,1.96,660
+3,13.48,1.67,2.64,22.5,89,2.6,1.1,.52,2.29,11.75,.57,1.78,620
+3,12.36,3.83,2.38,21,88,2.3,.92,.5,1.04,7.65,.56,1.58,520
+3,13.69,3.26,2.54,20,107,1.83,.56,.5,.8,5.88,.96,1.82,680
+3,12.85,3.27,2.58,22,106,1.65,.6,.6,.96,5.58,.87,2.11,570
+3,12.96,3.45,2.35,18.5,106,1.39,.7,.4,.94,5.28,.68,1.75,675
+3,13.78,2.76,2.3,22,90,1.35,.68,.41,1.03,9.58,.7,1.68,615
+3,13.73,4.36,2.26,22.5,88,1.28,.47,.52,1.15,6.62,.78,1.75,520
+3,13.45,3.7,2.6,23,111,1.7,.92,.43,1.46,10.68,.85,1.56,695
+3,12.82,3.37,2.3,19.5,88,1.48,.66,.4,.97,10.26,.72,1.75,685
+3,13.58,2.58,2.69,24.5,105,1.55,.84,.39,1.54,8.66,.74,1.8,750
+3,13.4,4.6,2.86,25,112,1.98,.96,.27,1.11,8.5,.67,1.92,630
+3,12.2,3.03,2.32,19,96,1.25,.49,.4,.73,5.5,.66,1.83,510
+3,12.77,2.39,2.28,19.5,86,1.39,.51,.48,.64,9.899999,.57,1.63,470
+3,14.16,2.51,2.48,20,91,1.68,.7,.44,1.24,9.7,.62,1.71,660
+3,13.71,5.65,2.45,20.5,95,1.68,.61,.52,1.06,7.7,.64,1.74,740
+3,13.4,3.91,2.48,23,102,1.8,.75,.43,1.41,7.3,.7,1.56,750
+3,13.27,4.28,2.26,20,120,1.59,.69,.43,1.35,10.2,.59,1.56,835
+3,13.17,2.59,2.37,20,120,1.65,.68,.53,1.46,9.3,.6,1.62,840
+3,14.13,4.1,2.74,24.5,96,2.05,.76,.56,1.35,9.2,.61,1.6,560

benchmark/tests/excel/.gitignore

@@ -0,0 +1 @@
+#

benchmark/tests/results/best_results_accuracy_STree.json

@@ -1 +1 @@
-{"balance-scale": [0.98, {"splitter": "best", "max_features": "auto"}, "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json"], "balloons": [0.86, {"C": 7, "gamma": 0.1, "kernel": "rbf", "max_iter": 10000.0, "multiclass_strategy": "ovr"}, "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"]}
+{"balance-scale": [0.98, {"splitter": "best", "max_features": "auto"}, "results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json"], "balloons": [0.86, {"C": 7, "gamma": 0.1, "kernel": "rbf", "max_iter": 10000, "multiclass_strategy": "ovr"}, "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"]}

benchmark/tests/results/grid_output_accuracy_STree.json

@@ -6,7 +6,7 @@
             "kernel": "liblinear",
             "multiclass_strategy": "ovr"
         },
-        "v. 1.2.4, Computed on Test on 2022-02-22 at 12:00:00 took 1s"
+        "v. 1.4.0, Computed on Test on 2022-02-22 at 12:00:00 took 1s"
     ],
     "balloons": [
         0.625,
@@ -15,6 +15,6 @@
             "kernel": "linear",
             "multiclass_strategy": "ovr"
         },
-        "v. 1.2.4, Computed on Test on 2022-02-22 at 12:00:00 took 1s"
+        "v. 1.4.0, Computed on Test on 2022-02-22 at 12:00:00 took 1s"
     ]
 }

benchmark/tests/results/results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json

@@ -1,57 +1 @@
-{
-  "score_name": "accuracy",
-  "title": "Gridsearched hyperparams v022.1b random_init",
-  "model": "ODTE",
-  "version": "0.3.2",
-  "stratified": false,
-  "folds": 5,
-  "date": "2022-04-20",
-  "time": "10:52:20",
-  "duration": 22591.471411943436,
-  "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1],
-  "platform": "Galgo",
-  "results": [
-    {
-      "dataset": "balance-scale",
-      "samples": 625,
-      "features": 4,
-      "classes": 3,
-      "hyperparameters": {
-        "base_estimator__C": 57,
-        "base_estimator__gamma": 0.1,
-        "base_estimator__kernel": "rbf",
-        "base_estimator__multiclass_strategy": "ovr",
-        "n_estimators": 100,
-        "n_jobs": -1
-      },
-      "nodes": 7.361199999999999,
-      "leaves": 4.180599999999999,
-      "depth": 3.536,
-      "score": 0.96352,
-      "score_std": 0.024949741481626608,
-      "time": 0.31663217544555666,
-      "time_std": 0.19918813895255585
-    },
-    {
-      "dataset": "balloons",
-      "samples": 16,
-      "features": 4,
-      "classes": 2,
-      "hyperparameters": {
-        "base_estimator__C": 5,
-        "base_estimator__gamma": 0.14,
-        "base_estimator__kernel": "rbf",
-        "base_estimator__multiclass_strategy": "ovr",
-        "n_estimators": 100,
-        "n_jobs": -1
-      },
-      "nodes": 2.9951999999999996,
-      "leaves": 1.9975999999999998,
-      "depth": 1.9975999999999998,
-      "score": 0.785,
-      "score_std": 0.2461311755051675,
-      "time": 0.11560620784759522,
-      "time_std": 0.012784241828599895
-    }
-  ]
-}
+{"score_name": "accuracy", "title": "Gridsearched hyperparams v022.1b random_init", "model": "ODTE", "version": "0.3.2", "language_version": "3.11x", "language": "Python", "stratified": false, "folds": 5, "date": "2022-04-20", "time": "10:52:20", "duration": 22591.471411943436, "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1], "platform": "Galgo", "results": [{"dataset": "balance-scale", "samples": 625, "features": 4, "classes": 3, "hyperparameters": {"base_estimator__C": 57, "base_estimator__gamma": 0.1, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr", "n_estimators": 100, "n_jobs": -1}, "nodes": 7.361199999999999, "leaves": 4.180599999999999, "depth": 3.536, "score": 0.96352, "score_std": 0.024949741481626608, "time": 0.31663217544555666, "time_std": 0.19918813895255585}, {"dataset": "balloons", "samples": 16, "features": 4, "classes": 2, "hyperparameters": {"base_estimator__C": 5, "base_estimator__gamma": 0.14, "base_estimator__kernel": "rbf", "base_estimator__multiclass_strategy": "ovr", "n_estimators": 100, "n_jobs": -1}, "nodes": 2.9951999999999996, "leaves": 1.9975999999999998, "depth": 1.9975999999999998, "score": 0.785, "score_std": 0.2461311755051675, "time": 0.11560620784759522, "time_std": 0.012784241828599895}], "discretized": false}

benchmark/tests/results/results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json

@@ -1,43 +1 @@
-{
-  "score_name": "accuracy",
-  "title": "Test default paramters with RandomForest",
-  "model": "RandomForest",
-  "version": "-",
-  "stratified": false,
-  "folds": 5,
-  "date": "2022-01-14",
-  "time": "12:39:30",
-  "duration": 272.7363500595093,
-  "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1],
-  "platform": "iMac27",
-  "results": [
-    {
-      "dataset": "balance-scale",
-      "samples": 625,
-      "features": 4,
-      "classes": 3,
-      "hyperparameters": {},
-      "nodes": 196.91440000000003,
-      "leaves": 98.42,
-      "depth": 10.681399999999998,
-      "score": 0.83616,
-      "score_std": 0.02649630917694009,
-      "time": 0.08222018241882324,
-      "time_std": 0.0013026326815120633
-    },
-    {
-      "dataset": "balloons",
-      "samples": 16,
-      "features": 4,
-      "classes": 2,
-      "hyperparameters": {},
-      "nodes": 9.110800000000001,
-      "leaves": 4.58,
-      "depth": 3.0982,
-      "score": 0.625,
-      "score_std": 0.24958298553119898,
-      "time": 0.07016648769378662,
-      "time_std": 0.002460508923990468
-    }
-  ]
-}
+{"score_name": "accuracy", "title": "Test default paramters with RandomForest", "model": "RandomForest", "version": "-", "language_version": "3.11x", "language": "Python", "stratified": false, "folds": 5, "date": "2022-01-14", "time": "12:39:30", "duration": 272.7363500595093, "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1], "platform": "iMac27", "results": [{"dataset": "balance-scale", "samples": 625, "features": 4, "classes": 3, "hyperparameters": {}, "nodes": 196.91440000000003, "leaves": 98.42, "depth": 10.681399999999998, "score": 0.83616, "score_std": 0.02649630917694009, "time": 0.08222018241882324, "time_std": 0.0013026326815120633}, {"dataset": "balloons", "samples": 16, "features": 4, "classes": 2, "hyperparameters": {}, "nodes": 9.110800000000001, "leaves": 4.58, "depth": 3.0982, "score": 0.625, "score_std": 0.24958298553119898, "time": 0.07016648769378662, "time_std": 0.002460508923990468}], "discretized": false}

benchmark/tests/results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json

@@ -1,55 +1 @@
-{
-  "score_name": "accuracy",
-  "model": "STree",
-  "stratified": false,
-  "folds": 5,
-  "date": "2021-09-30",
-  "time": "11:42:07",
-  "duration": 624.2505249977112,
-  "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1],
-  "platform": "iMac27",
-  "results": [
-    {
-      "dataset": "balance-scale",
-      "samples": 625,
-      "features": 4,
-      "classes": 3,
-      "hyperparameters": {
-        "C": 10000.0,
-        "gamma": 0.1,
-        "kernel": "rbf",
-        "max_iter": 10000.0,
-        "multiclass_strategy": "ovr"
-      },
-      "nodes": 7.0,
-      "leaves": 4.0,
-      "depth": 3.0,
-      "score": 0.97056,
-      "score_std": 0.015046806970251203,
-      "time": 0.01404867172241211,
-      "time_std": 0.002026269126958884
-    },
-    {
-      "dataset": "balloons",
-      "samples": 16,
-      "features": 4,
-      "classes": 2,
-      "hyperparameters": {
-        "C": 7,
-        "gamma": 0.1,
-        "kernel": "rbf",
-        "max_iter": 10000.0,
-        "multiclass_strategy": "ovr"
-      },
-      "nodes": 3.0,
-      "leaves": 2.0,
-      "depth": 2.0,
-      "score": 0.86,
-      "score_std": 0.28501461950807594,
-      "time": 0.0008541679382324218,
-      "time_std": 3.629469326417878e-5
-    }
-  ],
-  "title": "With gridsearched hyperparameters",
-  "version": "1.2.3"
-}
+{"score_name": "accuracy", "model": "STree", "stratified": false, "folds": 5, "language_version": "3.11x", "language": "Python", "date": "2021-09-30", "time": "11:42:07", "duration": 624.2505249977112, "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1], "platform": "iMac27", "results": [{"dataset": "balance-scale", "samples": 625, "features": 4, "classes": 3, "hyperparameters": {"C": 10000, "gamma": 0.1, "kernel": "rbf", "max_iter": 10000, "multiclass_strategy": "ovr"}, "nodes": 7.0, "leaves": 4.0, "depth": 3.0, "score": 0.97056, "score_std": 0.015046806970251203, "time": 0.01404867172241211, "time_std": 0.002026269126958884}, {"dataset": "balloons", "samples": 16, "features": 4, "classes": 2, "hyperparameters": {"C": 7, "gamma": 0.1, "kernel": "rbf", "max_iter": 10000, "multiclass_strategy": "ovr"}, "nodes": 3.0, "leaves": 2.0, "depth": 2.0, "score": 0.86, "score_std": 0.28501461950807594, "time": 0.0008541679382324218, "time_std": 3.629469326417878e-05}], "title": "With gridsearched hyperparameters", "version": "1.2.3", "discretized": false}

benchmark/tests/results/results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json

@@ -1,49 +1 @@
-{
-  "score_name": "accuracy",
-  "model": "STree",
-  "stratified": false,
-  "folds": 5,
-  "date": "2021-10-27",
-  "time": "09:40:40",
-  "duration": 3395.009148836136,
-  "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1],
-  "platform": "iMac27",
-  "results": [
-    {
-      "dataset": "balance-scale",
-      "samples": 625,
-      "features": 4,
-      "classes": 3,
-      "hyperparameters": {
-        "splitter": "best",
-        "max_features": "auto"
-      },
-      "nodes": 11.08,
-      "leaves": 5.9,
-      "depth": 5.9,
-      "score": 0.98,
-      "score_std": 0.001,
-      "time": 0.28520655155181884,
-      "time_std": 0.06031593282605064
-    },
-    {
-      "dataset": "balloons",
-      "samples": 16,
-      "features": 4,
-      "classes": 2,
-      "hyperparameters": {
-        "splitter": "best",
-        "max_features": "auto"
-      },
-      "nodes": 4.12,
-      "leaves": 2.56,
-      "depth": 2.56,
-      "score": 0.695,
-      "score_std": 0.2756860130252853,
-      "time": 0.021201000213623047,
-      "time_std": 0.003526023309468471
-    }
-  ],
-  "title": "default A",
-  "version": "1.2.3"
-}
+{"score_name": "accuracy", "model": "STree", "language": "Python", "language_version": "3.11x", "stratified": false, "folds": 5, "date": "2021-10-27", "time": "09:40:40", "duration": 3395.009148836136, "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1], "platform": "iMac27", "results": [{"dataset": "balance-scale", "samples": 625, "features": 4, "classes": 3, "hyperparameters": {"splitter": "best", "max_features": "auto"}, "nodes": 11.08, "leaves": 5.9, "depth": 5.9, "score": 0.98, "score_std": 0.001, "time": 0.28520655155181884, "time_std": 0.06031593282605064}, {"dataset": "balloons", "samples": 16, "features": 4, "classes": 2, "hyperparameters": {"splitter": "best", "max_features": "auto"}, "nodes": 4.12, "leaves": 2.56, "depth": 2.56, "score": 0.695, "score_std": 0.2756860130252853, "time": 0.021201000213623047, "time_std": 0.003526023309468471}], "title": "default A", "version": "1.2.3", "discretized": false}

benchmark/tests/results/results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json

@@ -1,49 +1 @@
-{
-  "score_name": "accuracy",
-  "model": "STree",
-  "stratified": false,
-  "folds": 5,
-  "date": "2021-11-01",
-  "time": "19:17:07",
-  "duration": 4115.042420864105,
-  "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1],
-  "platform": "macbook-pro",
-  "results": [
-    {
-      "dataset": "balance-scale",
-      "samples": 625,
-      "features": 4,
-      "classes": 3,
-      "hyperparameters": {
-        "max_features": "auto",
-        "splitter": "mutual"
-      },
-      "nodes": 18.78,
-      "leaves": 9.88,
-      "depth": 5.9,
-      "score": 0.97,
-      "score_std": 0.002,
-      "time": 0.23330417156219482,
-      "time_std": 0.048087665954193885
-    },
-    {
-      "dataset": "balloons",
-      "samples": 16,
-      "features": 4,
-      "classes": 2,
-      "hyperparameters": {
-        "max_features": "auto",
-        "splitter": "mutual"
-      },
-      "nodes": 4.72,
-      "leaves": 2.86,
-      "depth": 2.78,
-      "score": 0.5566666666666668,
-      "score_std": 0.2941277122460771,
-      "time": 0.021352062225341795,
-      "time_std": 0.005808742398555902
-    }
-  ],
-  "title": "default B",
-  "version": "1.2.3"
-}
+{"score_name": "accuracy", "model": "STree", "language_version": "3.11x", "language": "Python", "stratified": false, "folds": 5, "date": "2021-11-01", "time": "19:17:07", "duration": 4115.042420864105, "seeds": [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1], "platform": "macbook-pro", "results": [{"dataset": "balance-scale", "samples": 625, "features": 4, "classes": 3, "hyperparameters": {"max_features": "auto", "splitter": "mutual"}, "nodes": 18.78, "leaves": 9.88, "depth": 5.9, "score": 0.97, "score_std": 0.002, "time": 0.23330417156219482, "time_std": 0.048087665954193885}, {"dataset": "balloons", "samples": 16, "features": 4, "classes": 2, "hyperparameters": {"max_features": "auto", "splitter": "mutual"}, "nodes": 4.72, "leaves": 2.86, "depth": 2.78, "score": 0.5566666666666668, "score_std": 0.2941277122460771, "time": 0.021352062225341795, "time_std": 0.005808742398555902}], "title": "default B", "version": "1.2.3", "discretized": false}

benchmark/tests/scripts/Be_Benchmark_test.py

@@ -2,6 +2,7 @@ import os
 from openpyxl import load_workbook
 from ...Utils import NO_RESULTS, Folders, Files
 from ..TestBase import TestBase
+from ..._version import __version__
 
 
 class BeBenchmarkTest(TestBase):
@@ -15,16 +16,16 @@ class BeBenchmarkTest(TestBase):
             files.append(Files.exreport(score))
             files.append(Files.exreport_output(score))
             files.append(Files.exreport_err(score))
-        files.append(Files.exreport_excel(self.score))
         files.append(Files.exreport_pdf)
         files.append(Files.tex_output(self.score))
         self.remove_files(files, Folders.exreport)
+        self.remove_files([Files.exreport_excel(self.score)], Folders.excel)
         self.remove_files(files, ".")
         return super().tearDown()
 
     def test_be_benchmark_complete(self):
         stdout, stderr = self.execute_script(
-            "be_benchmark", ["-s", self.score, "-q", "1", "-t", "1", "-x", "1"]
+            "be_benchmark", ["-s", self.score, "-q", "-t", "-x"]
         )
         self.assertEqual(stderr.getvalue(), "")
         # Check output
@@ -40,16 +41,26 @@ class BeBenchmarkTest(TestBase):
         self.check_file_file(file_name, "exreport_tex")
         # Check excel file
         file_name = os.path.join(
-            Folders.exreport, Files.exreport_excel(self.score)
+            Folders.excel, Files.exreport_excel(self.score)
         )
         book = load_workbook(file_name)
+        replace = None
+        with_this = None
         for sheet_name in book.sheetnames:
             sheet = book[sheet_name]
-            self.check_excel_sheet(sheet, f"exreport_excel_{sheet_name}")
+            if sheet_name == "Datasets":
+                replace = self.benchmark_version
+                with_this = __version__
+            self.check_excel_sheet(
+                sheet,
+                f"exreport_excel_{sheet_name}",
+                replace=replace,
+                with_this=with_this,
+            )
 
     def test_be_benchmark_single(self):
         stdout, stderr = self.execute_script(
-            "be_benchmark", ["-s", self.score, "-q", "1"]
+            "be_benchmark", ["-s", self.score, "-q"]
         )
         self.assertEqual(stderr.getvalue(), "")
         # Check output

benchmark/tests/scripts/Be_Best_test.py

@@ -67,7 +67,7 @@ class BeBestTest(TestBase):
 
     def test_be_build_best_report(self):
         stdout, _ = self.execute_script(
-            "be_build_best", ["-s", "accuracy", "-m", "ODTE", "-r", "1"]
+            "be_build_best", ["-s", "accuracy", "-m", "ODTE", "-r"]
         )
         expected_data = {
             "balance-scale": [

benchmark/tests/scripts/Be_Grid_test.py

@@ -4,6 +4,10 @@ from ...Utils import Folders, Files
 from ..TestBase import TestBase
 
 
+def get_test():
+    return "hola"
+
+
 class BeGridTest(TestBase):
     def setUp(self):
         self.prepare_scripts_env()
@@ -65,7 +69,7 @@ class BeGridTest(TestBase):
     def test_be_grid_no_input(self):
         stdout, stderr = self.execute_script(
             "be_grid",
-            ["-m", "ODTE", "-s", "f1-weighted", "-q", "1"],
+            ["-m", "ODTE", "-s", "f1-weighted", "-q"],
         )
         self.assertEqual(stderr.getvalue(), "")
         grid_file = os.path.join(

benchmark/tests/scripts/Be_Init_Project_test.py

@@ -0,0 +1,67 @@
+import os
+from io import StringIO
+from unittest.mock import patch
+from ..TestBase import TestBase
+from ...Utils import Folders
+
+
+class BeInitProjectTest(TestBase):
+    def setUp(self):
+        self.prepare_scripts_env()
+
+    def tearDown(self):
+        if os.path.exists("test_project"):
+            os.system("rm -rf test_project")
+
+    def assertIsFile(self, file_name):
+        if not os.path.isfile(file_name):
+            raise AssertionError(f"File {str(file_name)} does not exist")
+
+    def assertIsFolder(self, path):
+        if not os.path.exists(path):
+            raise AssertionError(f"Folder {str(path)} does not exist")
+
+    def test_be_init_project(self):
+        test_project = "test_project"
+        stdout, stderr = self.execute_script("be_init_project", [test_project])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_init_project")
+        # check folders
+        expected = [
+            Folders.results,
+            Folders.hidden_results,
+            Folders.exreport,
+            Folders.report,
+            Folders.img,
+            Folders.excel,
+        ]
+        for folder in expected:
+            self.assertIsFolder(os.path.join(test_project, folder))
+        self.assertIsFile(os.path.join(test_project, ".env"))
+        os.system(f"rm -rf {test_project}")
+
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_init_project_no_arguments(self, stdout, stderr):
+        with self.assertRaises(SystemExit) as cm:
+            module = self.search_script("be_init_project")
+            module.main("")
+        self.assertEqual(cm.exception.code, 2)
+        self.check_output_file(stdout, "be_init_project_no_arguments")
+        self.assertEqual(stderr.getvalue(), "")
+
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_init_project_twice(self, stdout, stderr):
+        test_project = "test_project"
+        self.execute_script("be_init_project", [test_project])
+        with self.assertRaises(SystemExit) as cm:
+            module = self.search_script("be_init_project")
+            module.main([test_project])
+        self.assertEqual(cm.exception.code, 1)
+        self.assertEqual(
+            stderr.getvalue(),
+            f"Creating folder {test_project}\n"
+            f"[Errno 17] File exists: '{test_project}'\n",
+        )
+        self.assertEqual(stdout.getvalue(), "")

benchmark/tests/scripts/Be_List_test.py

@@ -1,5 +1,8 @@
 import os
-from ...Utils import Folders, NO_RESULTS
+import shutil
+from unittest.mock import patch
+from openpyxl import load_workbook
+from ...Utils import Folders, Files, NO_RESULTS
 from ..TestBase import TestBase
 
 
@@ -7,19 +10,94 @@ class BeListTest(TestBase):
     def setUp(self):
         self.prepare_scripts_env()
 
-    def test_be_list(self):
+    @patch("benchmark.Manager.get_input", return_value="q")
+    def test_be_list(self, input_data):
         stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "summary_list_model")
+        self.check_output_file(stdout, "be_list_model")
 
-    def test_be_list_no_data(self):
+    @patch("benchmark.Manager.get_input", side_effect=iter(["x", "q"]))
+    def test_be_list_invalid_option(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_model_invalid")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["0", "q"]))
+    def test_be_list_report(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_report")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["r", "q"]))
+    def test_be_list_twice(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_model_2")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["e 2", "q"]))
+    def test_be_list_report_excel(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_report_excel")
+
+        book = load_workbook(os.path.join(Folders.excel, Files.be_list_excel))
+        sheet = book["STree"]
+        self.check_excel_sheet(sheet, "excel")
+
+    @patch(
+        "benchmark.Manager.get_input",
+        side_effect=iter(["e 2", "e 1", "q"]),
+    )
+    def test_be_list_report_excel_twice(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_report_excel_2")
+        book = load_workbook(os.path.join(Folders.excel, Files.be_list_excel))
+        sheet = book["STree"]
+        self.check_excel_sheet(sheet, "excel")
+        sheet = book["STree2"]
+        self.check_excel_sheet(sheet, "excel2")
+
+    @patch("benchmark.Manager.get_input", return_value="q")
+    def test_be_list_no_data(self, input_data):
         stdout, stderr = self.execute_script(
             "be_list", ["-m", "Wodt", "-s", "f1-macro"]
         )
         self.assertEqual(stderr.getvalue(), "")
         self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n")
 
-    def test_be_list_nan(self):
+    @patch(
+        "benchmark.Manager.get_input",
+        side_effect=iter(["d 0", "y", "", "q"]),
+    )
+    # @patch("benchmark.ResultsBase.get_input", side_effect=iter(["q"]))
+    def test_be_list_delete(self, input_data):
+        def copy_files(source_folder, target_folder, file_name):
+            source = os.path.join(source_folder, file_name)
+            target = os.path.join(target_folder, file_name)
+            shutil.copyfile(source, target)
+
+        file_name = (
+            "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:"
+            "35_0.json"
+        )
+        # move nan result from hidden to results
+        copy_files(Folders.hidden_results, Folders.results, file_name)
+        try:
+            # list and delete result
+            stdout, stderr = self.execute_script("be_list", "")
+            self.assertEqual(stderr.getvalue(), "")
+            self.check_output_file(stdout, "be_list_delete")
+        except Exception:
+            # delete the result copied if be_list couldn't
+            os.unlink(os.path.join(Folders.results, file_name))
+            self.fail("test_be_list_delete() should not raise exception")
+
+    @patch(
+        "benchmark.Manager.get_input",
+        side_effect=iter(["h 0", "y", "", "q"]),
+    )
+    def test_be_list_hide(self, input_data):
         def swap_files(source_folder, target_folder, file_name):
             source = os.path.join(source_folder, file_name)
             target = os.path.join(target_folder, file_name)
@@ -32,19 +110,44 @@ class BeListTest(TestBase):
         # move nan result from hidden to results
         swap_files(Folders.hidden_results, Folders.results, file_name)
         try:
-            # list and move nan result to hidden
-            stdout, stderr = self.execute_script("be_list", ["--nan", "1"])
+            # list and move nan result to hidden again
+            stdout, stderr = self.execute_script("be_list", "")
             self.assertEqual(stderr.getvalue(), "")
-            self.check_output_file(stdout, "be_list_nan")
+            self.check_output_file(stdout, "be_list_hide")
         except Exception:
-            # move back nan result file if be_list couldn't
+            # delete the result copied if be_list couldn't
             swap_files(Folders.results, Folders.hidden_results, file_name)
-            self.fail("test_be_list_nan() should not raise exception")
+            self.fail("test_be_list_hide() should not raise exception")
 
-    def test_be_list_nan_no_nan(self):
-        stdout, stderr = self.execute_script("be_list", ["--nan", "1"])
+    @patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "q"]))
+    def test_be_list_already_hidden(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "be_list_no_nan")
+        self.check_output_file(stdout, "be_list_already_hidden")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["h 0", "n", "q"]))
+    def test_be_list_dont_hide(self, input_data):
+        stdout, stderr = self.execute_script("be_list", "")
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_default")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["q"]))
+    def test_be_list_hidden_nan(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_hidden_nan")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["q"]))
+    def test_be_list_hidden(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_hidden")
+
+    @patch("benchmark.Manager.get_input", side_effect=iter(["0", "q"]))
+    def test_be_list_compare(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--compare"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_compare_fault")
 
     def test_be_no_env(self):
         path = os.getcwd()

benchmark/tests/scripts/Be_Main_test.py

@@ -1,4 +1,5 @@
 import os
+import json
 from io import StringIO
 from unittest.mock import patch
 from ...Results import Report
@@ -24,19 +25,21 @@ class BeMainTest(TestBase):
         self.check_output_lines(
             stdout=stdout,
             file_name="be_main_dataset",
-            lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13],
+            lines_to_compare=[0, 2, 3, 5, 6, 7, 8, 9, 11, 12, 13, 14],
         )
 
     def test_be_main_complete(self):
         stdout, _ = self.execute_script(
             "be_main",
-            ["-s", self.score, "-m", "STree", "--title", "test", "-r", "1"],
+            ["-s", self.score, "-m", "STree", "--title", "test", "-r"],
         )
         # keep the report name to delete it after
         report_name = stdout.getvalue().splitlines()[-1].split("in ")[1]
         self.files.append(report_name)
         self.check_output_lines(
-            stdout, "be_main_complete", [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14]
+            stdout,
+            "be_main_complete",
+            [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14, 15],
         )
 
     def test_be_main_no_report(self):
@@ -66,10 +69,8 @@ class BeMainTest(TestBase):
                 "STree",
                 "--title",
                 "test",
-                "-f",
-                "1",
+                "-b",
                 "-r",
-                "1",
             ],
         )
         # keep the report name to delete it after
@@ -79,6 +80,48 @@ class BeMainTest(TestBase):
             stdout, "be_main_best", [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14]
         )
 
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_main_incompatible_params(self, stdout, stderr):
+        m1 = (
+            "be_main: error: argument -b/--best_paramfile: not allowed with "
+            "argument -p/--hyperparameters"
+        )
+        m2 = (
+            "be_main: error: argument -g/--grid_paramfile: not allowed with "
+            "argument -p/--hyperparameters"
+        )
+        m3 = (
+            "be_main: error: argument -g/--grid_paramfile: not allowed with "
+            "argument -p/--hyperparameters"
+        )
+        m4 = m1
+        p0 = [
+            "-s",
+            self.score,
+            "-m",
+            "SVC",
+            "--title",
+            "test",
+        ]
+        pset = json.dumps(dict(C=17))
+        p1 = p0.copy()
+        p1.extend(["-p", pset, "-b"])
+        p2 = p0.copy()
+        p2.extend(["-p", pset, "-g"])
+        p3 = p0.copy()
+        p3.extend(["-p", pset, "-g", "-b"])
+        p4 = p0.copy()
+        p4.extend(["-b", "-g"])
+        parameters = [(p1, m1), (p2, m2), (p3, m3), (p4, m4)]
+        for parameter, message in parameters:
+            with self.assertRaises(SystemExit) as msg:
+                module = self.search_script("be_main")
+                module.main(parameter)
+            self.assertEqual(msg.exception.code, 2)
+            self.assertEqual(stderr.getvalue(), "")
+            self.assertRegex(stdout.getvalue(), message)
+
     def test_be_main_best_params_non_existent(self):
         model = "GBC"
         stdout, stderr = self.execute_script(
@@ -90,10 +133,8 @@ class BeMainTest(TestBase):
                 model,
                 "--title",
                 "test",
-                "-f",
-                "1",
+                "-b",
                 "-r",
-                "1",
             ],
         )
         self.assertEqual(stderr.getvalue(), "")
@@ -117,9 +158,7 @@ class BeMainTest(TestBase):
                 "--title",
                 "test",
                 "-g",
-                "1",
                 "-r",
-                "1",
             ],
         )
         self.assertEqual(stderr.getvalue(), "")
@@ -142,9 +181,7 @@ class BeMainTest(TestBase):
                 "--title",
                 "test",
                 "-g",
-                "1",
                 "-r",
-                "1",
             ],
         )
         # keep the report name to delete it after

benchmark/tests/scripts/Be_Print_Strees_test.py

@@ -18,7 +18,7 @@ class BePrintStrees(TestBase):
         for name in self.datasets:
             stdout, _ = self.execute_script(
                 "be_print_strees",
-                ["-d", name, "-q", "1"],
+                ["-d", name, "-q"],
             )
             file_name = os.path.join(Folders.img, f"stree_{name}.png")
             self.files.append(file_name)
@@ -27,13 +27,13 @@ class BePrintStrees(TestBase):
                 stdout.getvalue(), f"File {file_name} generated\n"
             )
             computed_size = os.path.getsize(file_name)
-            self.assertGreater(computed_size, 25000)
+            self.assertGreater(computed_size, 24500)
 
     def test_be_print_strees_dataset_color(self):
         for name in self.datasets:
             stdout, _ = self.execute_script(
                 "be_print_strees",
-                ["-d", name, "-q", "1", "-c", "1"],
+                ["-d", name, "-q", "-c"],
             )
             file_name = os.path.join(Folders.img, f"stree_{name}.png")
             self.files.append(file_name)

benchmark/tests/scripts/Be_Report_test.py

@@ -1,7 +1,10 @@
 import os
 from openpyxl import load_workbook
-from ...Utils import Folders
+from io import StringIO
+from unittest.mock import patch
+from ...Utils import Folders, Files
 from ..TestBase import TestBase
+from ..._version import __version__
 
 
 class BeReportTest(TestBase):
@@ -10,10 +13,17 @@ class BeReportTest(TestBase):
 
     def tearDown(self) -> None:
         files = [
-            "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.sql",
             "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.xlsx",
         ]
         self.remove_files(files, Folders.results)
+        self.remove_files(
+            [Files.datasets_report_excel],
+            os.path.join(os.getcwd(), Folders.excel),
+        )
+        files = [
+            "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.sql",
+        ]
+        self.remove_files(files, Folders.sql)
         return super().tearDown()
 
     def test_be_report(self):
@@ -21,58 +31,113 @@ class BeReportTest(TestBase):
             "results",
             "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
         )
-        stdout, stderr = self.execute_script("be_report", ["-f", file_name])
+        stdout, stderr = self.execute_script("be_report", ["file", file_name])
         self.assertEqual(stderr.getvalue(), "")
         self.check_output_file(stdout, "report")
 
     def test_be_report_not_found(self):
-        stdout, stderr = self.execute_script("be_report", ["-f", "unknown"])
+        stdout, stderr = self.execute_script("be_report", ["file", "unknown"])
         self.assertEqual(stderr.getvalue(), "")
         self.assertEqual(stdout.getvalue(), "unknown does not exists!\n")
 
-    def test_be_report_compare(self):
+    def test_be_report_compared(self):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"
         stdout, stderr = self.execute_script(
-            "be_report", ["-f", file_name, "-c", "1"]
+            "be_report", ["file", file_name, "-c"]
         )
         self.assertEqual(stderr.getvalue(), "")
         self.check_output_file(stdout, "report_compared")
 
     def test_be_report_datatsets(self):
-        stdout, stderr = self.execute_script("be_report", [])
+        stdout, stderr = self.execute_script("be_report", ["datasets"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "report_datasets")
+        file_name = f"report_datasets{self.ext}"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        output_text = stdout.getvalue().splitlines()
+        for line, index in zip(expected.splitlines(), range(len(expected))):
+            if self.benchmark_version in line:
+                # replace benchmark version
+                line = self.replace_benchmark_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])
+
+    def test_be_report_datasets_excel(self):
+        stdout, stderr = self.execute_script("be_report", ["datasets", "-x"])
+        self.assertEqual(stderr.getvalue(), "")
+        file_name = f"report_datasets{self.ext}"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        output_text = stdout.getvalue().splitlines()
+        for line, index in zip(expected.splitlines(), range(len(expected))):
+            if self.benchmark_version in line:
+                # replace benchmark version
+                line = self.replace_benchmark_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])
+        file_name = os.path.join(
+            os.getcwd(), Folders.excel, Files.datasets_report_excel
+        )
+        book = load_workbook(file_name)
+        sheet = book["Datasets"]
+        self.check_excel_sheet(
+            sheet,
+            "exreport_excel_Datasets",
+            replace=self.benchmark_version,
+            with_this=__version__,
+        )
 
     def test_be_report_best(self):
         stdout, stderr = self.execute_script(
-            "be_report", ["-s", "accuracy", "-m", "STree", "-b", "1"]
+            "be_report", ["best", "-s", "accuracy", "-m", "STree"]
         )
         self.assertEqual(stderr.getvalue(), "")
         self.check_output_file(stdout, "report_best")
 
     def test_be_report_grid(self):
         stdout, stderr = self.execute_script(
-            "be_report", ["-s", "accuracy", "-m", "STree", "-g", "1"]
+            "be_report", ["grid", "-s", "accuracy", "-m", "STree"]
         )
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "report_grid")
+        file_name = "report_grid.test"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read().splitlines()
+        output_text = stdout.getvalue().splitlines()
+        # Compare replacing STree version
+        for line, index in zip(expected, range(len(expected))):
+            if "1.2.4" in line:
+                # replace STree version
+                line = self.replace_STree_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])
 
-    def test_be_report_best_both(self):
-        stdout, stderr = self.execute_script(
-            "be_report",
-            ["-s", "accuracy", "-m", "STree", "-b", "1", "-g", "1"],
-        )
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_report_unknown_subcommand(self, stderr):
+        with self.assertRaises(SystemExit) as msg:
+            module = self.search_script("be_report")
+            module.main(["unknown"])
+        self.assertEqual(msg.exception.code, 2)
+        self.check_output_file(stderr, "report_unknown_subcommand")
+
+    def test_be_report_without_subcommand(self):
+        stdout, stderr = self.execute_script("be_report", "")
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "report_best")
+        self.maxDiff = None
+        # Can't use check_output_file because of the width of the console
+        # output is different in different environments
+        file_name = "report_without_subcommand" + self.ext
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        if expected == stdout.getvalue():
+            self.assertEqual(stdout.getvalue(), expected)
+        else:
+            self.check_output_file(stdout, "report_without_subcommand2")
 
     def test_be_report_excel_compared(self):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"
         stdout, stderr = self.execute_script(
             "be_report",
-            ["-f", file_name, "-x", "1", "-c", "1"],
+            ["file", file_name, "-x", "-c"],
         )
         file_name = os.path.join(
-            Folders.results, file_name.replace(".json", ".xlsx")
+            Folders.excel, file_name.replace(Files.report_ext, ".xlsx")
         )
         book = load_workbook(file_name)
         sheet = book["STree"]
@@ -84,10 +149,10 @@ class BeReportTest(TestBase):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"
         stdout, stderr = self.execute_script(
             "be_report",
-            ["-f", file_name, "-x", "1"],
+            ["file", file_name, "-x"],
         )
         file_name = os.path.join(
-            Folders.results, file_name.replace(".json", ".xlsx")
+            Folders.excel, file_name.replace(Files.report_ext, ".xlsx")
         )
         book = load_workbook(file_name)
         sheet = book["STree"]
@@ -99,10 +164,10 @@ class BeReportTest(TestBase):
         file_name = "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json"
         stdout, stderr = self.execute_script(
             "be_report",
-            ["-f", file_name, "-q", "1"],
+            ["file", file_name, "-q"],
         )
         file_name = os.path.join(
-            Folders.results, file_name.replace(".json", ".sql")
+            Folders.sql, file_name.replace(Files.report_ext, ".sql")
         )
         self.check_file_file(file_name, "sql")
         self.assertEqual(stderr.getvalue(), "")

benchmark/tests/sql/.gitignore

@@ -0,0 +1 @@
+#

benchmark/tests/test_files/be_build_best_report.test

@@ -7,5 +7,5 @@ Dataset                        Score    File/Message
 balance-scale                  0.963520 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json                       {'base_estimator__C': 57, 'base_estimator__gamma': 0.1, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1} 
 balloons                       0.785000 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json                       {'base_estimator__C': 5, 'base_estimator__gamma': 0.14, 'base_estimator__kernel': 'rbf', 'base_estimator__multiclass_strategy': 'ovr', 'n_estimators': 100, 'n_jobs': -1} 
 ******************************************************************************************************************************************************************
-* Scores compared to stree_default accuracy (liblinear-ovr) .:  0.0434                                                                                           *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0434                                                                                                  *
 ******************************************************************************************************************************************************************

benchmark/tests/test_files/be_build_grid.test

@@ -6,13 +6,13 @@
         "n_estimators": [
             100
         ],
-        "base_estimator__C": [
+        "estimator__C": [
             1.0
         ],
-        "base_estimator__kernel": [
+        "estimator__kernel": [
             "linear"
         ],
-        "base_estimator__multiclass_strategy": [
+        "estimator__multiclass_strategy": [
             "ovo"
         ]
     },
@@ -23,7 +23,7 @@
         "n_estimators": [
             100
         ],
-        "base_estimator__C": [
+        "estimator__C": [
             0.001,
             0.0275,
             0.05,
@@ -36,10 +36,10 @@
             7,
             10000.0
         ],
-        "base_estimator__kernel": [
+        "estimator__kernel": [
             "liblinear"
         ],
-        "base_estimator__multiclass_strategy": [
+        "estimator__multiclass_strategy": [
             "ovr"
         ]
     },
@@ -50,7 +50,7 @@
         "n_estimators": [
             100
         ],
-        "base_estimator__C": [
+        "estimator__C": [
             0.05,
             1.0,
             1.05,
@@ -62,7 +62,7 @@
             57,
             10000.0
         ],
-        "base_estimator__gamma": [
+        "estimator__gamma": [
             0.001,
             0.1,
             0.14,
@@ -70,10 +70,10 @@
             "auto",
             "scale"
         ],
-        "base_estimator__kernel": [
+        "estimator__kernel": [
             "rbf"
         ],
-        "base_estimator__multiclass_strategy": [
+        "estimator__multiclass_strategy": [
             "ovr"
         ]
     },
@@ -84,20 +84,20 @@
         "n_estimators": [
             100
         ],
-        "base_estimator__C": [
+        "estimator__C": [
             0.05,
             0.2,
             1.0,
             8.25
         ],
-        "base_estimator__gamma": [
+        "estimator__gamma": [
             0.1,
             "scale"
         ],
-        "base_estimator__kernel": [
+        "estimator__kernel": [
             "poly"
         ],
-        "base_estimator__multiclass_strategy": [
+        "estimator__multiclass_strategy": [
             "ovo",
             "ovr"
         ]

benchmark/tests/test_files/be_init_project.test

@@ -0,0 +1,12 @@
+Creating folder test_project
+Creating folder test_project/results
+Creating folder test_project/hidden_results
+Creating folder test_project/exreport
+Creating folder test_project/exreport/exreport_output
+Creating folder test_project/img
+Creating folder test_project/excel
+Creating folder test_project/sql
+Done!
+Please, edit .env file with your settings and add a datasets folder
+with an all.txt file with the datasets you want to use.
+In that folder you have to include all the datasets you'll use.

benchmark/tests/test_files/be_init_project_no_arguments.test

@@ -0,0 +1,2 @@
+usage: be_init_project [-h] project_name
+be_init_project: error: the following arguments are required: project_name

benchmark/tests/test_files/be_list_already_hidden.test

@@ -0,0 +1,5 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default
+Already hidden

benchmark/tests/test_files/be_list_compare_fault.test

@@ -0,0 +1,8 @@
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+results/best_results_accuracy_ODTE.json does not exist

benchmark/tests/test_files/be_list_default.test

@@ -0,0 +1,7 @@
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_delete.test

@@ -0,0 +1,16 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= ============================================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json            0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json   0.03627   0.076 Test default paramters with RandomForest
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json     0.03790   1.143 default B
+  4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json          0.04158   0.943 default A
+  5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json          0.04544   0.173 With gridsearched hyperparameters
+Deleting results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters