Update bayesclass models

Add excel output of reports of be_list

.github/workflows/main.yml

@@ -12,11 +12,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-latest]
+        os: [ubuntu-latest]
-        python: ["3.10", "3.11"]
+        python: ["3.10"]
-        exclude:
-          - os: macos-latest
-            python: "3.11"
 
     steps:
       - uses: actions/checkout@v3
@@ -46,6 +43,7 @@ jobs:
           pip install -q --upgrade pip
           pip install -q -r requirements.txt
           pip install -q --upgrade codecov coverage black flake8
+          git clone https://github.com/Doctorado-ML/bayesclass.git
       - name: Lint
         run: |
           black --check --diff benchmark

README.md

@@ -1,7 +1,7 @@
 [![CI](https://github.com/Doctorado-ML/benchmark/actions/workflows/main.yml/badge.svg)](https://github.com/Doctorado-ML/benchmark/actions/workflows/main.yml)
 [![codecov](https://codecov.io/gh/Doctorado-ML/benchmark/branch/main/graph/badge.svg?token=ZRP937NDSG)](https://codecov.io/gh/Doctorado-ML/benchmark)
-[![Quality Gate Status](https://haystack.rmontanana.es:25000/api/project_badges/measure?project=benchmark&metric=alert_status&token=336a6e501988888543c3153baa91bad4b9914dd2)](http://haystack.local:25000/dashboard?id=benchmark)
+[![Quality Gate Status](https://haystack.rmontanana.es:25000/api/project_badges/measure?project=benchmark&metric=alert_status&token=336a6e501988888543c3153baa91bad4b9914dd2)](https://haystack.rmontanana.es:25000/dashboard?id=benchmark)
-[![Technical Debt](https://haystack.rmontanana.es:25000/api/project_badges/measure?project=benchmark&metric=sqale_index&token=336a6e501988888543c3153baa91bad4b9914dd2)](http://haystack.local:25000/dashboard?id=benchmark)
+[![Technical Debt](https://haystack.rmontanana.es:25000/api/project_badges/measure?project=benchmark&metric=sqale_index&token=336a6e501988888543c3153baa91bad4b9914dd2)](https://haystack.rmontanana.es:25000/dashboard?id=benchmark)
 ![https://img.shields.io/badge/python-3.8%2B-blue](https://img.shields.io/badge/python-3.8%2B-brightgreen)
 
 # benchmark

benchmark/Arguments.py

@@ -36,6 +36,7 @@ class EnvDefault(argparse.Action):
         self, envvar, required=True, default=None, mandatory=False, **kwargs
     ):
         self._args = EnvData.load()
+        self._overrides = {}
         if required and not mandatory:
             default = self._args[envvar]
             required = False
@@ -47,24 +48,27 @@ class EnvDefault(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
-class Arguments:
+class Arguments(argparse.ArgumentParser):
-    def __init__(self):
+    def __init__(self, *args, **kwargs):
-        self.ap = argparse.ArgumentParser()
+        super().__init__(*args, **kwargs)
         models_data = Models.define_models(random_state=0)
+        self._overrides = {}
+        self._subparser = None
         self.parameters = {
-            "best": [
+            "best_paramfile": [
-                ("-b", "--best"),
+                ("-b", "--best_paramfile"),
                 {
-                    "type": str,
+                    "action": "store_true",
                     "required": False,
-                    "help": "best results of models",
+                    "default": False,
+                    "help": "Use best hyperparams file?",
                 },
             ],
             "color": [
                 ("-c", "--color"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "use colors for the tree",
                 },
@@ -72,8 +76,9 @@ class Arguments:
             "compare": [
                 ("-c", "--compare"),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "required": False,
+                    "default": False,
                     "help": "Compare accuracy with best results",
                 },
             ],
@@ -81,6 +86,8 @@ class Arguments:
                 ("-d", "--dataset"),
                 {
                     "type": str,
+                    "envvar": "dataset",  # for compatiblity with EnvDefault
+                    "action": EnvDefault,
                     "required": False,
                     "help": "dataset to work with",
                 },
@@ -88,38 +95,26 @@ class Arguments:
             "excel": [
                 ("-x", "--excel"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "Generate Excel File",
                 },
             ],
-            "file": [
-                ("-f", "--file"),
-                {"type": str, "required": False, "help": "Result file"},
-            ],
-            "grid": [
-                ("-g", "--grid"),
-                {
-                    "type": str,
-                    "required": False,
-                    "help": "grid results of model",
-                },
-            ],
             "grid_paramfile": [
                 ("-g", "--grid_paramfile"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
-                    "help": "Use best hyperparams file?",
+                    "help": "Use grid output hyperparams file?",
                 },
             ],
             "hidden": [
                 ("--hidden",),
                 {
-                    "type": str,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "Show hidden results",
                 },
@@ -140,8 +135,8 @@ class Arguments:
             "lose": [
                 ("-l", "--lose"),
                 {
-                    "type": bool,
                     "default": False,
+                    "action": "store_true",
                     "required": False,
                     "help": "show lose results",
                 },
@@ -178,9 +173,10 @@ class Arguments:
             "nan": [
                 ("--nan",),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "required": False,
-                    "help": "Move nan results to hidden folder",
+                    "default": False,
+                    "help": "List nan results to hidden folder",
                 },
             ],
             "number": [
@@ -202,15 +198,6 @@ class Arguments:
                     "help": "number of folds",
                 },
             ],
-            "paramfile": [
-                ("-f", "--paramfile"),
-                {
-                    "type": bool,
-                    "required": False,
-                    "default": False,
-                    "help": "Use best hyperparams file?",
-                },
-            ],
             "platform": [
                 ("-P", "--platform"),
                 {
@@ -224,7 +211,7 @@ class Arguments:
             "quiet": [
                 ("-q", "--quiet"),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "required": False,
                     "default": False,
                 },
@@ -232,7 +219,7 @@ class Arguments:
             "report": [
                 ("-r", "--report"),
                 {
-                    "type": bool,
+                    "action": "store_true",
                     "default": False,
                     "required": False,
                     "help": "Report results",
@@ -250,14 +237,18 @@ class Arguments:
             ],
             "sql": [
                 ("-q", "--sql"),
-                {"type": bool, "required": False, "help": "Generate SQL File"},
+                {
+                    "required": False,
+                    "action": "store_true",
+                    "default": False,
+                    "help": "Generate SQL File",
+                },
             ],
             "stratified": [
                 ("-t", "--stratified"),
                 {
                     "action": EnvDefault,
                     "envvar": "stratified",
-                    "type": str,
                     "required": True,
                     "help": "Stratified",
                 },
@@ -265,8 +256,8 @@ class Arguments:
             "tex_output": [
                 ("-t", "--tex-output"),
                 {
-                    "type": bool,
                     "required": False,
+                    "action": "store_true",
                     "default": False,
                     "help": "Generate Tex file with the table",
                 },
@@ -278,8 +269,8 @@ class Arguments:
             "win": [
                 ("-w", "--win"),
                 {
-                    "type": bool,
                     "default": False,
+                    "action": "store_true",
                     "required": False,
                     "help": "show win results",
                 },
@@ -287,12 +278,43 @@ class Arguments:
         }
 
     def xset(self, *arg_name, **kwargs):
-        names, default = self.parameters[arg_name[0]]
+        names, parameters = self.parameters[arg_name[0]]
-        self.ap.add_argument(
+        if "overrides" in kwargs:
+            self._overrides[names[0]] = (kwargs["overrides"], kwargs["const"])
+            del kwargs["overrides"]
+        self.add_argument(
             *names,
-            **{**default, **kwargs},
+            **{**parameters, **kwargs},
         )
         return self
 
+    def add_subparser(
+        self, dest="subcommand", help_text="help for subcommand"
+    ):
+        self._subparser = self.add_subparsers(dest=dest, help=help_text)
+
+    def add_subparsers_options(self, subparser, arguments):
+        command, help_text = subparser
+        parser = self._subparser.add_parser(command, help=help_text)
+        for name, args in arguments:
+            try:
+                names, parameters = self.parameters[name]
+            except KeyError:
+                names = (name,)
+                parameters = {}
+            # Order of args is important
+            parser.add_argument(*names, **{**args, **parameters})
+
+    def add_exclusive(self, hyperparameters, required=False):
+        group = self.add_mutually_exclusive_group(required=required)
+        for name in hyperparameters:
+            names, parameters = self.parameters[name]
+            group.add_argument(*names, **parameters)
+
     def parse(self, args=None):
-        return self.ap.parse_args(args)
+        for key, (dest_key, value) in self._overrides.items():
+            if args is None:
+                args = sys.argv[1:]
+            if key in args:
+                args.extend((f"--{dest_key}", value))
+        return super().parse_args(args)

benchmark/Datasets.py

@@ -1,4 +1,5 @@
 import os
+from types import SimpleNamespace
 import pandas as pd
 import numpy as np
 from scipy.io import arff
@@ -26,18 +27,18 @@ class DatasetsArff:
     def folder():
         return "datasets"
 
-    def load(self, name, class_name, dataframe):
+    def load(self, name, class_name):
         file_name = os.path.join(self.folder(), self.dataset_names(name))
         data = arff.loadarff(file_name)
         df = pd.DataFrame(data[0])
         df.dropna(axis=0, how="any", inplace=True)
+        self.dataset = df
         X = df.drop(class_name, axis=1)
         self.features = X.columns
         self.class_name = class_name
         y, _ = pd.factorize(df[class_name])
-        df[class_name] = y
         X = X.to_numpy()
-        return df if dataframe else (X, y)
+        return X, y
 
 
 class DatasetsTanveer:
@@ -56,8 +57,12 @@ class DatasetsTanveer:
             sep="\t",
             index_col=0,
         )
-        X = data.drop("clase", axis=1).to_numpy()
+        X = data.drop("clase", axis=1)
+        self.features = X.columns
+        X = X.to_numpy()
         y = data["clase"].to_numpy()
+        self.dataset = data
+        self.class_name = "clase"
         return X, y
 
 
@@ -78,8 +83,11 @@ class DatasetsSurcov:
         )
         data.dropna(axis=0, how="any", inplace=True)
         self.columns = data.columns
-        col_list = ["class"]
+        X = data.drop(["class"], axis=1)
-        X = data.drop(col_list, axis=1).to_numpy()
+        self.features = X.columns
+        self.class_name = "class"
+        self.dataset = data
+        X = X.to_numpy()
         y = data["class"].to_numpy()
         return X, y
 
@@ -87,50 +95,48 @@ class DatasetsSurcov:
 class Datasets:
     def __init__(self, dataset_name=None):
         envData = EnvData.load()
-        class_name = getattr(
+        # DatasetsSurcov, DatasetsTanveer, DatasetsArff,...
+        source_name = getattr(
             __import__(__name__),
             f"Datasets{envData['source_data']}",
         )
-        self.load = (
+        self.discretize = envData["discretize"] == "1"
-            self.load_discretized
+        self.dataset = source_name()
-            if envData["discretize"] == "1"
-            else self.load_continuous
-        )
-        self.dataset = class_name()
         self.class_names = []
-        self._load_names()
+        self.data_sets = []
-        if dataset_name is not None:
+        # initialize self.class_names & self.data_sets
-            try:
+        class_names, sets = self._init_names(dataset_name)
-                class_name = self.class_names[
+        self.class_names = class_names
-                    self.data_sets.index(dataset_name)
+        self.data_sets = sets
-                ]
-                self.class_names = [class_name]
-            except ValueError:
-                raise ValueError(f"Unknown dataset: {dataset_name}")
-            self.data_sets = [dataset_name]
 
-    def _load_names(self):
+    def _init_names(self, dataset_name):
         file_name = os.path.join(self.dataset.folder(), Files.index)
         default_class = "class"
         with open(file_name) as f:
-            self.data_sets = f.read().splitlines()
+            sets = f.read().splitlines()
-            self.class_names = [default_class] * len(self.data_sets)
+            class_names = [default_class] * len(sets)
-        if "," in self.data_sets[0]:
+        if "," in sets[0]:
             result = []
             class_names = []
-            for data in self.data_sets:
+            for data in sets:
                 name, class_name = data.split(",")
                 result.append(name)
                 class_names.append(class_name)
-            self.data_sets = result
+            sets = result
-            self.class_names = class_names
+        # Set as dataset list the dataset passed as argument
+        if dataset_name is None:
+            return class_names, sets
+        try:
+            class_name = class_names[sets.index(dataset_name)]
+        except ValueError:
+            raise ValueError(f"Unknown dataset: {dataset_name}")
+        return [class_name], [dataset_name]
 
     def get_attributes(self, name):
-        class Attributes:
+        tmp = self.discretize
-            pass
+        self.discretize = False
-
+        X, y = self.load(name)
-        X, y = self.load_continuous(name)
+        attr = SimpleNamespace()
-        attr = Attributes()
         values, counts = np.unique(y, return_counts=True)
         comp = ""
         sep = ""
@@ -141,6 +147,7 @@ class Datasets:
         attr.classes = len(np.unique(y))
         attr.samples = X.shape[0]
         attr.features = X.shape[1]
+        self.discretize = tmp
         return attr
 
     def get_features(self):
@@ -149,14 +156,25 @@ class Datasets:
     def get_class_name(self):
         return self.dataset.class_name
 
-    def load_continuous(self, name, dataframe=False):
+    def get_dataset(self):
+        return self.dataset.dataset
+
+    def load(self, name, dataframe=False):
         try:
             class_name = self.class_names[self.data_sets.index(name)]
-            return self.dataset.load(name, class_name, dataframe)
+            X, y = self.dataset.load(name, class_name)
+            if self.discretize:
+                X = self.discretize_dataset(X, y)
+                dataset = pd.DataFrame(X, columns=self.get_features())
+                dataset[self.get_class_name()] = y
+                self.dataset.dataset = dataset
+            if dataframe:
+                return self.get_dataset()
+            return X, y
         except (ValueError, FileNotFoundError):
             raise ValueError(f"Unknown dataset: {name}")
 
-    def discretize(self, X, y):
+    def discretize_dataset(self, X, y):
         """Supervised discretization with Fayyad and Irani's MDLP algorithm.
 
         Parameters
@@ -170,16 +188,9 @@ class Datasets:
         -------
         tuple (X, y) of numpy.ndarray
         """
-        discretiz = MDLP()
+        discretiz = MDLP(random_state=17, dtype=np.int32)
         Xdisc = discretiz.fit_transform(X, y)
-        return Xdisc.astype(int), y.astype(int)
+        return Xdisc
-
-    def load_discretized(self, name, dataframe=False):
-        X, y = self.load_continuous(name)
-        X, y = self.discretize(X, y)
-        dataset = pd.DataFrame(X, columns=self.get_features())
-        dataset[self.get_class_name()] = y
-        return dataset if dataframe else X, y
 
     def __iter__(self) -> Diterator:
         return Diterator(self.data_sets)

benchmark/Models.py

@@ -8,7 +8,7 @@ from sklearn.ensemble import (
 )
 from sklearn.svm import SVC
 from stree import Stree
-from bayesclass import TAN
+from bayesclass.clfs import TAN, KDB, AODE
 from wodt import Wodt
 from odte import Odte
 from xgboost import XGBClassifier
@@ -22,6 +22,8 @@ class Models:
         return {
             "STree": Stree(random_state=random_state),
             "TAN": TAN(random_state=random_state),
+            "KDB": KDB(k=3),
+            "AODE": AODE(random_state=random_state),
             "Cart": DecisionTreeClassifier(random_state=random_state),
             "ExtraTree": ExtraTreeClassifier(random_state=random_state),
             "Wodt": Wodt(random_state=random_state),

benchmark/Results.py

@@ -1,12 +1,14 @@
 import os
 import sys
 from operator import itemgetter
+from types import SimpleNamespace
 import math
 import json
 import abc
 import shutil
 import subprocess
 import xlsxwriter
+from xlsxwriter.exceptions import DuplicateWorksheetName
 import numpy as np
 from .Experiments import BestResults
 from .Datasets import Datasets
@@ -21,6 +23,10 @@ from .Utils import (
 from ._version import __version__
 
 
+def get_input(message="", is_test=False):
+    return "test" if is_test else input(message)
+
+
 class BestResultsEver:
     def __init__(self):
         self.data = {}
@@ -35,7 +41,7 @@ class BestResultsEver:
         ]
         self.data["Arff"]["accuracy"] = [
             "STree_default (linear-ovo)",
-            22.063496,
+            22.109799,
         ]
 
     def get_name_value(self, key, score):
@@ -125,7 +131,7 @@ class BaseReport(abc.ABC):
 
 
 class Report(BaseReport):
-    header_lengths = [30, 6, 5, 3, 7, 7, 7, 15, 16, 15]
+    header_lengths = [30, 6, 5, 3, 7, 7, 7, 15, 17, 15]
     header_cols = [
         "Dataset",
         "Sampl.",
@@ -184,7 +190,7 @@ class Report(BaseReport):
         )
         i += 1
         print(
-            f"{result['time']:9.6f}±{result['time_std']:6.4f} ",
+            f"{result['time']:10.6f}±{result['time_std']:6.4f} ",
             end="",
         )
         i += 1
@@ -246,7 +252,7 @@ class ReportBest(BaseReport):
         "Hyperparameters",
     ]
 
-    def __init__(self, score, model, best, grid):
+    def __init__(self, score, model, best):
         name = (
             Files.best_results(score, model)
             if best
@@ -254,7 +260,6 @@ class ReportBest(BaseReport):
         )
         file_name = os.path.join(Folders.results, name)
         self.best = best
-        self.grid = grid
         self.score_name = score
         self.model = model
         super().__init__(file_name, best_file=True)
@@ -328,7 +333,17 @@ class Excel(BaseReport):
         else:
             self.book = book
             self.close = False
-        self.sheet = self.book.add_worksheet(self.data["model"])
+        suffix = ""
+        num = 1
+        while True:
+            try:
+                self.sheet = self.book.add_worksheet(
+                    self.data["model"] + suffix
+                )
+                break
+            except DuplicateWorksheetName:
+                num += 1
+                suffix = str(num)
         self.max_hyper_width = 0
         self.col_hyperparams = 0
 
@@ -1338,12 +1353,14 @@ class StubReport(BaseReport):
 
 
 class Summary:
-    def __init__(self, hidden=False) -> None:
+    def __init__(self, hidden=False, compare=False) -> None:
         self.results = Files().get_all_results(hidden=hidden)
         self.data = []
+        self.data_filtered = []
         self.datasets = {}
         self.models = set()
         self.hidden = hidden
+        self.compare = compare
 
     def get_models(self):
         return sorted(self.models)
@@ -1386,18 +1403,15 @@ class Summary:
                 self.data.append(entry)
 
     def get_results_criteria(
-        self,
+        self, score, model, input_data, sort_key, number, nan=False
-        score,
-        model,
-        input_data,
-        sort_key,
-        number,
     ):
         data = self.data.copy() if input_data is None else input_data
         if score:
             data = [x for x in data if x["score"] == score]
         if model:
             data = [x for x in data if x["model"] == model]
+        if nan:
+            data = [x for x in data if x["metric"] != x["metric"]]
         keys = (
             itemgetter(sort_key, "time")
             if sort_key == "date"
@@ -1415,15 +1429,17 @@ class Summary:
         input_data=None,
         sort_key="date",
         number=0,
+        nan=False,
     ) -> None:
         """Print the list of results"""
-        data = self.get_results_criteria(
+        if self.data_filtered == []:
-            score, model, input_data, sort_key, number
+            self.data_filtered = self.get_results_criteria(
-        )
+                score, model, input_data, sort_key, number, nan=nan
-        if data == []:
+            )
+        if self.data_filtered == []:
             raise ValueError(NO_RESULTS)
-        max_file = max(len(x["file"]) for x in data)
+        max_file = max(len(x["file"]) for x in self.data_filtered)
-        max_title = max(len(x["title"]) for x in data)
+        max_title = max(len(x["title"]) for x in self.data_filtered)
         if self.hidden:
             color1 = TextColor.GREEN
             color2 = TextColor.YELLOW
@@ -1432,10 +1448,11 @@ class Summary:
             color2 = TextColor.LINE2
         print(color1, end="")
         print(
-            f"{'Date':10s} {'File':{max_file}s} {'Score':8s} {'Time(h)':7s} "
+            f" #  {'Date':10s} {'File':{max_file}s} {'Score':8s} "
-            f"{'Title':s}"
+            f"{'Time(h)':7s} {'Title':s}"
         )
         print(
+            "===",
             "=" * 10
             + " "
             + "=" * max_file
@@ -1444,21 +1461,115 @@ class Summary:
             + " "
             + "=" * 7
             + " "
-            + "=" * max_title
+            + "=" * max_title,
         )
         print(
             "\n".join(
                 [
-                    (color2 if n % 2 == 0 else color1)
+                    (color2 if n % 2 == 0 else color1) + f"{n:3d} "
-                    + f"{x['date']} {x['file']:{max_file}s} "
+                    f"{x['date']} {x['file']:{max_file}s} "
                     f"{x['metric']:8.5f} "
                     f"{x['duration']/3600:7.3f} "
                     f"{x['title']}"
-                    for n, x in enumerate(data)
+                    for n, x in enumerate(self.data_filtered)
                 ]
             )
         )
 
+    def manage_results(self):
+        """Manage results showed in the summary
+        return True if excel file is created False otherwise
+        """
+
+        def process_file(num, command, path):
+            num = int(num)
+            name = self.data_filtered[num]["file"]
+            file_name_result = os.path.join(path, name)
+            verb1, verb2 = (
+                ("delete", "Deleting")
+                if command == cmd.delete
+                else (
+                    "hide",
+                    "Hiding",
+                )
+            )
+            conf_message = (
+                TextColor.RED
+                + f"Are you sure to {verb1} {file_name_result} (y/n)? "
+            )
+            confirm = get_input(message=conf_message)
+            if confirm == "y":
+                print(TextColor.YELLOW + f"{verb2} {file_name_result}")
+                if command == cmd.delete:
+                    os.unlink(file_name_result)
+                else:
+                    os.rename(
+                        os.path.join(Folders.results, name),
+                        os.path.join(Folders.hidden_results, name),
+                    )
+                self.data_filtered.pop(num)
+                get_input(message="Press enter to continue")
+                self.list_results()
+
+        cmd = SimpleNamespace(
+            quit="q", relist="r", delete="d", hide="h", excel="e"
+        )
+        message = (
+            TextColor.ENDC
+            + f"Choose option {str(cmd).replace('namespace', '')}: "
+        )
+        path = Folders.hidden_results if self.hidden else Folders.results
+        book = None
+        max_value = len(self.data)
+        while True:
+            match get_input(message=message).split():
+                case [cmd.relist]:
+                    self.list_results()
+                case [cmd.quit]:
+                    if book is not None:
+                        book.close()
+                        return True
+                    return False
+                case [cmd.hide, num] if num.isdigit() and int(num) < max_value:
+                    if self.hidden:
+                        print("Already hidden")
+                    else:
+                        process_file(num, path=path, command=cmd.hide)
+                case [cmd.delete, num] if num.isdigit() and int(
+                    num
+                ) < max_value:
+                    process_file(num=num, path=path, command=cmd.delete)
+                case [cmd.excel, num] if num.isdigit() and int(
+                    num
+                ) < max_value:
+                    # Add to excel file result #num
+                    num = int(num)
+                    file_name_result = os.path.join(
+                        path, self.data_filtered[num]["file"]
+                    )
+                    if book is None:
+                        file_name = Files.be_list_excel
+                        book = xlsxwriter.Workbook(
+                            file_name, {"nan_inf_to_errors": True}
+                        )
+                    excel = Excel(
+                        file_name=file_name_result,
+                        book=book,
+                        compare=self.compare,
+                    )
+                    excel.report()
+                    print(f"Added {file_name_result} to {Files.be_list_excel}")
+                case [num] if num.isdigit() and int(num) < max_value:
+                    # Report the result #num
+                    num = int(num)
+                    file_name_result = os.path.join(
+                        path, self.data_filtered[num]["file"]
+                    )
+                    rep = Report(file_name_result, compare=self.compare)
+                    rep.report()
+                case _:
+                    print("Invalid option. Try again!")
+
     def show_result(self, data: dict, title: str = "") -> None:
         def whites(n: int) -> str:
             return " " * n + color1 + "*"

benchmark/Utils.py

@@ -28,6 +28,7 @@ class Files:
     benchmark_r = "benchmark.r"
     dot_env = ".env"
     datasets_report_excel = "ReportDatasets.xlsx"
+    be_list_excel = "some_results.xlsx"
 
     @staticmethod
     def exreport_output(score):
@@ -145,3 +146,7 @@ class TextColor:
     ENDC = "\033[0m"
     BOLD = "\033[1m"
     UNDERLINE = "\033[4m"
+    WHITE = "\033[97m"
+    GREY = "\033[90m"
+    BLACK = "\033[90m"
+    DEFAULT = "\033[99m"

benchmark/_version

@@ -1 +0,0 @@
-__version__ = "0.7.1"

benchmark/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0"
+__version__ = "0.4.0"

benchmark/scripts/be_benchmark.py

@@ -5,7 +5,7 @@ from benchmark.Arguments import Arguments
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    arguments = Arguments(prog="be_benchmark")
     arguments.xset("score").xset("excel").xset("tex_output").xset("quiet")
     args = arguments.parse(args_test)
     benchmark = Benchmark(score=args.score, visualize=not args.quiet)

benchmark/scripts/be_build_best.py

@@ -21,5 +21,5 @@ def main(args_test=None):
         print(e)
     else:
         if args.report:
-            report = ReportBest(args.score, args.model, best=True, grid=False)
+            report = ReportBest(args.score, args.model, best=True)
             report.report()

benchmark/scripts/be_init_project.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+import os
+from benchmark.Utils import Files, Folders
+from benchmark.Arguments import Arguments
+
+
+def main(args_test=None):
+    arguments = Arguments(prog="be_init_project")
+    arguments.add_argument("project_name", help="Project name")
+    args = arguments.parse(args_test)
+    folders = []
+    folders.append(args.project_name)
+    folders.append(os.path.join(args.project_name, Folders.results))
+    folders.append(os.path.join(args.project_name, Folders.hidden_results))
+    folders.append(os.path.join(args.project_name, Folders.exreport))
+    folders.append(os.path.join(args.project_name, Folders.report))
+    folders.append(os.path.join(args.project_name, Folders.img))
+    try:
+        for folder in folders:
+            print(f"Creating folder {folder}")
+            os.makedirs(folder)
+    except FileExistsError as e:
+        print(e)
+        exit(1)
+    env_src = os.path.join(Folders.src(), "..", f"{Files.dot_env}.dist")
+    env_to = os.path.join(args.project_name, Files.dot_env)
+    os.system(f"cp {env_src} {env_to}")
+    print("Done!")
+    print(
+        "Please, edit .env file with your settings and add a datasets folder"
+    )
+    print("with an all.txt file with the datasets you want to use.")
+    print("In that folder you have to include all the datasets you'll use.")

benchmark/scripts/be_list.py

@@ -1,7 +1,6 @@
 #! /usr/bin/env python
-import os
 from benchmark.Results import Summary
-from benchmark.Utils import Folders
+from benchmark.Utils import Files
 from benchmark.Arguments import Arguments
 
 """List experiments of a model
@@ -9,11 +8,12 @@ from benchmark.Arguments import Arguments
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    arguments = Arguments(prog="be_list")
     arguments.xset("number").xset("model", required=False).xset("key")
-    arguments.xset("hidden").xset("nan").xset("score", required=False)
+    arguments.xset("score", required=False).xset("compare").xset("hidden")
+    arguments.xset("nan")
     args = arguments.parse(args_test)
-    data = Summary(hidden=args.hidden)
+    data = Summary(hidden=args.hidden, compare=args.compare)
     data.acquire()
     try:
         data.list_results(
@@ -21,33 +21,12 @@ def main(args_test=None):
             model=args.model,
             sort_key=args.key,
             number=args.number,
+            nan=args.nan,
         )
     except ValueError as e:
         print(e)
-    else:
+        return
-        if args.nan:
+    excel_generated = data.manage_results()
-            results_nan = []
+    if excel_generated:
-            results = data.get_results_criteria(
+        print(f"Generated file: {Files.be_list_excel}")
-                score=args.score,
+        Files.open(Files.be_list_excel, test=args_test is not None)
-                model=args.model,
-                input_data=None,
-                sort_key=args.key,
-                number=args.number,
-            )
-            for result in results:
-                if result["metric"] != result["metric"]:
-                    results_nan.append(result)
-            if results_nan != []:
-                print(
-                    "\n"
-                    + "*" * 30
-                    + " Results with nan moved to hidden "
-                    + "*" * 30
-                )
-                data.list_results(input_data=results_nan)
-                for result in results_nan:
-                    name = result["file"]
-                    os.rename(
-                        os.path.join(Folders.results, name),
-                        os.path.join(Folders.hidden_results, name),
-                    )

benchmark/scripts/be_main.py

@@ -10,15 +10,20 @@ from benchmark.Arguments import Arguments
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    arguments = Arguments(prog="be_main")
     arguments.xset("stratified").xset("score").xset("model", mandatory=True)
     arguments.xset("n_folds").xset("platform").xset("quiet").xset("title")
-    arguments.xset("hyperparameters").xset("paramfile").xset("report")
+    arguments.xset("report")
-    arguments.xset("grid_paramfile").xset("dataset")
+    arguments.add_exclusive(
+        ["grid_paramfile", "best_paramfile", "hyperparameters"]
+    )
+    arguments.xset(
+        "dataset", overrides="title", const="Test with only one dataset"
+    )
     args = arguments.parse(args_test)
     report = args.report or args.dataset is not None
     if args.grid_paramfile:
-        args.paramfile = False
+        args.best_paramfile = False
     try:
         job = Experiment(
             score_name=args.score,
@@ -26,7 +31,7 @@ def main(args_test=None):
             stratified=args.stratified,
             datasets=Datasets(dataset_name=args.dataset),
             hyperparams_dict=args.hyperparameters,
-            hyperparams_file=args.paramfile,
+            hyperparams_file=args.best_paramfile,
             grid_paramfile=args.grid_paramfile,
             progress_bar=not args.quiet,
             platform=args.platform,

benchmark/scripts/be_report.py

@@ -5,46 +5,79 @@ from benchmark.Arguments import Arguments
 
 
 """Build report on screen of a result file, optionally generate excel and sql
-file, and can compare results of report with best results obtained by model
+file, and can compare results of report wibth best results obtained by model
 If no argument is set, displays the datasets and its characteristics
 """
 
 
 def main(args_test=None):
-    arguments = Arguments()
+    is_test = args_test is not None
-    arguments.xset("file").xset("excel").xset("sql").xset("compare")
+    arguments = Arguments(prog="be_report")
-    arguments.xset("best").xset("grid").xset("model", required=False)
+    arguments.add_subparser()
-    arguments.xset("score", required=False)
+    arguments.add_subparsers_options(
+        (
+            "best",
+            "Report best results obtained by any model/score. "
+            "See be_build_best",
+        ),
+        [
+            ("model", dict(required=False)),
+            ("score", dict(required=False)),
+        ],
+    )
+    arguments.add_subparsers_options(
+        (
+            "grid",
+            "Report grid results obtained by any model/score. "
+            "See be_build_grid",
+        ),
+        [
+            ("model", dict(required=False)),
+            ("score", dict(required=False)),
+        ],
+    )
+    arguments.add_subparsers_options(
+        ("file", "Report file results"),
+        [
+            ("file_name", {}),
+            ("excel", {}),
+            ("sql", {}),
+            ("compare", {}),
+        ],
+    )
+    arguments.add_subparsers_options(
+        ("datasets", "Report datasets information"),
+        [
+            ("excel", {}),
+        ],
+    )
     args = arguments.parse(args_test)
-    if args.best:
+    match args.subcommand:
-        args.grid = None
+        case "best" | "grid":
-    if args.grid:
+            best = args.subcommand == "best"
-        args.best = None
+            report = ReportBest(args.score, args.model, best)
-    if args.file is None and args.best is None and args.grid is None:
-        report = ReportDatasets(args.excel)
-        report.report()
-        if args.excel:
-            is_test = args_test is not None
-            Files.open(report.get_file_name(), is_test)
-    else:
-        if args.best is not None or args.grid is not None:
-            report = ReportBest(args.score, args.model, args.best, args.grid)
             report.report()
-        else:
+        case "file":
             try:
-                report = Report(args.file, args.compare)
+                report = Report(args.file_name, args.compare)
+                report.report()
             except FileNotFoundError as e:
                 print(e)
-            else:
+                return
-                report.report()
+            if args.sql:
-                if args.excel:
+                sql = SQL(args.file_name)
-                    excel = Excel(
+                sql.report()
-                        file_name=args.file,
+            if args.excel:
-                        compare=args.compare,
+                excel = Excel(
-                    )
+                    file_name=args.file_name,
-                    excel.report()
+                    compare=args.compare,
-                    is_test = args_test is not None
+                )
-                    Files.open(excel.get_file_name(), is_test)
+                excel.report()
-                if args.sql:
+                Files.open(excel.get_file_name(), is_test)
-                    sql = SQL(args.file)
+        case "datasets":
-                    sql.report()
+            report = ReportDatasets(args.excel)
+            report.report()
+            if args.excel:
+                Files.open(report.get_file_name(), is_test)
+        case _:
+            arguments.print_help()

benchmark/tests/.gitignore

@@ -1 +1,2 @@
 ReportDatasets.xlsx 
+some_results.xlsx

benchmark/tests/Arguments_test.py

@@ -24,13 +24,11 @@ class ArgumentsTest(TestBase):
 
     def test_parameters(self):
         expected_parameters = {
-            "best": ("-b", "--best"),
+            "best_paramfile": ("-b", "--best_paramfile"),
             "color": ("-c", "--color"),
             "compare": ("-c", "--compare"),
             "dataset": ("-d", "--dataset"),
             "excel": ("-x", "--excel"),
-            "file": ("-f", "--file"),
-            "grid": ("-g", "--grid"),
             "grid_paramfile": ("-g", "--grid_paramfile"),
             "hidden": ("--hidden",),
             "hyperparameters": ("-p", "--hyperparameters"),
@@ -42,7 +40,6 @@ class ArgumentsTest(TestBase):
             "nan": ("--nan",),
             "number": ("-n", "--number"),
             "n_folds": ("-n", "--n_folds"),
-            "paramfile": ("-f", "--paramfile"),
             "platform": ("-P", "--platform"),
             "quiet": ("-q", "--quiet"),
             "report": ("-r", "--report"),
@@ -98,3 +95,27 @@ class ArgumentsTest(TestBase):
         finally:
             os.chdir(path)
         self.assertEqual(stderr.getvalue(), f"{NO_ENV}\n")
+
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_overrides(self, stderr):
+        arguments = self.build_args()
+        arguments.xset("title")
+        arguments.xset("dataset", overrides="title", const="sample text")
+        test_args = ["-n", "3", "-m", "SVC", "-k", "1", "-d", "dataset"]
+        args = arguments.parse(test_args)
+        self.assertEqual(stderr.getvalue(), "")
+        self.assertEqual(args.title, "sample text")
+
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_overrides_no_args(self, stderr):
+        arguments = self.build_args()
+        arguments.xset("title")
+        arguments.xset("dataset", overrides="title", const="sample text")
+        test_args = None
+        with self.assertRaises(SystemExit):
+            arguments.parse(test_args)
+        self.assertRegexpMatches(
+            stderr.getvalue(),
+            r"error: the following arguments are required: -m/--model, "
+            "-k/--key, --title",
+        )

benchmark/tests/Benchmark_test.py

@@ -5,6 +5,7 @@ from openpyxl import load_workbook
 from .TestBase import TestBase
 from ..Utils import Folders, Files, NO_RESULTS
 from ..Results import Benchmark
+from .._version import __version__
 
 
 class BenchmarkTest(TestBase):
@@ -107,6 +108,16 @@ class BenchmarkTest(TestBase):
         benchmark.excel()
         file_name = benchmark.get_excel_file_name()
         book = load_workbook(file_name)
+        replace = None
+        with_this = None
         for sheet_name in book.sheetnames:
             sheet = book[sheet_name]
-            self.check_excel_sheet(sheet, f"exreport_excel_{sheet_name}")
+            if sheet_name == "Datasets":
+                replace = self.benchmark_version
+                with_this = __version__
+            self.check_excel_sheet(
+                sheet,
+                f"exreport_excel_{sheet_name}",
+                replace=replace,
+                with_this=with_this,
+            )

benchmark/tests/Dataset_test.py

@@ -30,6 +30,19 @@ class DatasetTest(TestBase):
         expected = [271, 314, 171]
         self.assertSequenceEqual(Randomized.seeds(), expected)
 
+    def test_load_dataframe(self):
+        self.set_env(".env.arff")
+        dt = Datasets()
+        X, y = dt.load("iris", dataframe=False)
+        dataset = dt.load("iris", dataframe=True)
+        class_name = dt.get_class_name()
+        features = dt.get_features()
+        self.assertListEqual(y.tolist(), dataset[class_name].tolist())
+        for i in range(len(features)):
+            self.assertListEqual(
+                X[:, i].tolist(), dataset[features[i]].tolist()
+            )
+
     def test_Datasets_iterator(self):
         test = {
             ".env.dist": ["balance-scale", "balloons"],

benchmark/tests/Report_test.py

@@ -2,11 +2,14 @@ import os
 from io import StringIO
 from unittest.mock import patch
 from .TestBase import TestBase
-from ..Results import Report, BaseReport, ReportBest, ReportDatasets
+from ..Results import Report, BaseReport, ReportBest, ReportDatasets, get_input
 from ..Utils import Symbols
 
 
 class ReportTest(TestBase):
+    def test_get_input(self):
+        self.assertEqual(get_input(is_test=True), "test")
+
     def test_BaseReport(self):
         with patch.multiple(BaseReport, __abstractmethods__=set()):
             file_name = os.path.join(
@@ -66,13 +69,13 @@ class ReportTest(TestBase):
             _ = Report("unknown_file")
 
     def test_report_best(self):
-        report = ReportBest("accuracy", "STree", best=True, grid=False)
+        report = ReportBest("accuracy", "STree", best=True)
         with patch(self.output, new=StringIO()) as stdout:
             report.report()
         self.check_output_file(stdout, "report_best")
 
     def test_report_grid(self):
-        report = ReportBest("accuracy", "STree", best=False, grid=True)
+        report = ReportBest("accuracy", "STree", best=False)
         with patch(self.output, new=StringIO()) as stdout:
             report.report()
         file_name = "report_grid.test"
@@ -81,20 +84,22 @@ class ReportTest(TestBase):
         output_text = stdout.getvalue().splitlines()
         # Compare replacing STree version
         for line, index in zip(expected, range(len(expected))):
-            if "1.2.4" in line:
+            if self.stree_version in line:
                 # replace STree version
                 line = self.replace_STree_version(line, output_text, index)
 
             self.assertEqual(line, output_text[index])
 
-    def test_report_best_both(self):
-        report = ReportBest("accuracy", "STree", best=True, grid=True)
-        with patch(self.output, new=StringIO()) as stdout:
-            report.report()
-        self.check_output_file(stdout, "report_best")
-
     @patch("sys.stdout", new_callable=StringIO)
     def test_report_datasets(self, mock_output):
         report = ReportDatasets()
         report.report()
-        self.check_output_file(mock_output, "report_datasets")
+        file_name = f"report_datasets{self.ext}"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        output_text = mock_output.getvalue().splitlines()
+        for line, index in zip(expected.splitlines(), range(len(expected))):
+            if self.benchmark_version in line:
+                # replace benchmark version
+                line = self.replace_benchmark_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])

benchmark/tests/TestBase.py

@@ -15,6 +15,8 @@ class TestBase(unittest.TestCase):
         self.test_files = "test_files"
         self.output = "sys.stdout"
         self.ext = ".test"
+        self.benchmark_version = "0.2.0"
+        self.stree_version = "1.2.4"
         super().__init__(*args, **kwargs)
 
     def remove_files(self, files, folder):
@@ -31,7 +33,9 @@ class TestBase(unittest.TestCase):
                     if value is not None:
                         print(f'{row};{col};"{value}"', file=f)
 
-    def check_excel_sheet(self, sheet, file_name):
+    def check_excel_sheet(
+        self, sheet, file_name, replace=None, with_this=None
+    ):
         file_name += self.ext
         with open(os.path.join(self.test_files, file_name), "r") as f:
             expected = csv.reader(f, delimiter=";")
@@ -43,6 +47,9 @@ class TestBase(unittest.TestCase):
                         value = float(value)
                     except ValueError:
                         pass
+                if replace is not None and isinstance(value, str):
+                    if replace in value:
+                        value = value.replace(replace, with_this)
                 self.assertEqual(sheet.cell(int(row), int(col)).value, value)
 
     def check_output_file(self, output, file_name):
@@ -51,10 +58,15 @@ class TestBase(unittest.TestCase):
             expected = f.read()
         self.assertEqual(output.getvalue(), expected)
 
-    @staticmethod
+    def replace_STree_version(self, line, output, index):
-    def replace_STree_version(line, output, index):
+        idx = line.find(self.stree_version)
-        idx = line.find("1.2.4")
+        return line.replace(self.stree_version, output[index][idx : idx + 5])
-        return line.replace("1.2.4", output[index][idx : idx + 5])
+
+    def replace_benchmark_version(self, line, output, index):
+        idx = line.find(self.benchmark_version)
+        return line.replace(
+            self.benchmark_version, output[index][idx : idx + 5]
+        )
 
     def check_file_file(self, computed_file, expected_file):
         with open(computed_file) as f:

benchmark/tests/__init__.py

@@ -13,6 +13,7 @@ from .PairCheck_test import PairCheckTest
 from .Arguments_test import ArgumentsTest
 from .scripts.Be_Pair_check_test import BePairCheckTest
 from .scripts.Be_List_test import BeListTest
+from .scripts.Be_Init_Project_test import BeInitProjectTest
 from .scripts.Be_Report_test import BeReportTest
 from .scripts.Be_Summary_test import BeSummaryTest
 from .scripts.Be_Grid_test import BeGridTest

benchmark/tests/scripts/Be_Benchmark_test.py

@@ -2,6 +2,7 @@ import os
 from openpyxl import load_workbook
 from ...Utils import NO_RESULTS, Folders, Files
 from ..TestBase import TestBase
+from ..._version import __version__
 
 
 class BeBenchmarkTest(TestBase):
@@ -24,7 +25,7 @@ class BeBenchmarkTest(TestBase):
 
     def test_be_benchmark_complete(self):
         stdout, stderr = self.execute_script(
-            "be_benchmark", ["-s", self.score, "-q", "1", "-t", "1", "-x", "1"]
+            "be_benchmark", ["-s", self.score, "-q", "-t", "-x"]
         )
         self.assertEqual(stderr.getvalue(), "")
         # Check output
@@ -43,13 +44,23 @@ class BeBenchmarkTest(TestBase):
             Folders.exreport, Files.exreport_excel(self.score)
         )
         book = load_workbook(file_name)
+        replace = None
+        with_this = None
         for sheet_name in book.sheetnames:
             sheet = book[sheet_name]
-            self.check_excel_sheet(sheet, f"exreport_excel_{sheet_name}")
+            if sheet_name == "Datasets":
+                replace = self.benchmark_version
+                with_this = __version__
+            self.check_excel_sheet(
+                sheet,
+                f"exreport_excel_{sheet_name}",
+                replace=replace,
+                with_this=with_this,
+            )
 
     def test_be_benchmark_single(self):
         stdout, stderr = self.execute_script(
-            "be_benchmark", ["-s", self.score, "-q", "1"]
+            "be_benchmark", ["-s", self.score, "-q"]
         )
         self.assertEqual(stderr.getvalue(), "")
         # Check output

benchmark/tests/scripts/Be_Best_test.py

@@ -67,7 +67,7 @@ class BeBestTest(TestBase):
 
     def test_be_build_best_report(self):
         stdout, _ = self.execute_script(
-            "be_build_best", ["-s", "accuracy", "-m", "ODTE", "-r", "1"]
+            "be_build_best", ["-s", "accuracy", "-m", "ODTE", "-r"]
         )
         expected_data = {
             "balance-scale": [

benchmark/tests/scripts/Be_Grid_test.py

@@ -4,6 +4,10 @@ from ...Utils import Folders, Files
 from ..TestBase import TestBase
 
 
+def get_test():
+    return "hola"
+
+
 class BeGridTest(TestBase):
     def setUp(self):
         self.prepare_scripts_env()
@@ -65,7 +69,7 @@ class BeGridTest(TestBase):
     def test_be_grid_no_input(self):
         stdout, stderr = self.execute_script(
             "be_grid",
-            ["-m", "ODTE", "-s", "f1-weighted", "-q", "1"],
+            ["-m", "ODTE", "-s", "f1-weighted", "-q"],
         )
         self.assertEqual(stderr.getvalue(), "")
         grid_file = os.path.join(

benchmark/tests/scripts/Be_Init_Project_test.py

@@ -0,0 +1,66 @@
+import os
+from io import StringIO
+from unittest.mock import patch
+from ..TestBase import TestBase
+from ...Utils import Folders
+
+
+class BeInitProjectTest(TestBase):
+    def setUp(self):
+        self.prepare_scripts_env()
+
+    def tearDown(self):
+        if os.path.exists("test_project"):
+            os.system("rm -rf test_project")
+
+    def assertIsFile(self, file_name):
+        if not os.path.isfile(file_name):
+            raise AssertionError(f"File {str(file_name)} does not exist")
+
+    def assertIsFolder(self, path):
+        if not os.path.exists(path):
+            raise AssertionError(f"Folder {str(path)} does not exist")
+
+    def test_be_init_project(self):
+        test_project = "test_project"
+        stdout, stderr = self.execute_script("be_init_project", [test_project])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_init_project")
+        # check folders
+        expected = [
+            Folders.results,
+            Folders.hidden_results,
+            Folders.exreport,
+            Folders.report,
+            Folders.img,
+        ]
+        for folder in expected:
+            self.assertIsFolder(os.path.join(test_project, folder))
+        self.assertIsFile(os.path.join(test_project, ".env"))
+        os.system(f"rm -rf {test_project}")
+
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_init_project_no_arguments(self, stdout, stderr):
+        with self.assertRaises(SystemExit) as cm:
+            module = self.search_script("be_init_project")
+            module.main("")
+        self.assertEqual(cm.exception.code, 2)
+        self.check_output_file(stdout, "be_init_project_no_arguments")
+        self.assertEqual(stderr.getvalue(), "")
+
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_init_project_twice(self, stdout, stderr):
+        test_project = "test_project"
+        self.execute_script("be_init_project", [test_project])
+        with self.assertRaises(SystemExit) as cm:
+            module = self.search_script("be_init_project")
+            module.main([test_project])
+        self.assertEqual(cm.exception.code, 1)
+        self.assertEqual(
+            stderr.getvalue(),
+            f"Creating folder {test_project}\n"
+            f"[Errno 17] File exists: '{test_project}'\n",
+        )
+        self.assertEqual(stdout.getvalue(), "")

benchmark/tests/scripts/Be_List_test.py

@@ -1,5 +1,8 @@
 import os
-from ...Utils import Folders, NO_RESULTS
+import shutil
+from unittest.mock import patch
+from openpyxl import load_workbook
+from ...Utils import Folders, Files, NO_RESULTS
 from ..TestBase import TestBase
 
 
@@ -7,19 +10,90 @@ class BeListTest(TestBase):
     def setUp(self):
         self.prepare_scripts_env()
 
-    def test_be_list(self):
+    @patch("benchmark.Results.get_input", return_value="q")
+    def test_be_list(self, input_data):
         stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "summary_list_model")
+        self.check_output_file(stdout, "be_list_model")
 
-    def test_be_list_no_data(self):
+    @patch("benchmark.Results.get_input", side_effect=iter(["x", "q"]))
+    def test_be_list_invalid_option(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_model_invalid")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["0", "q"]))
+    def test_be_list_report(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_report")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["r", "q"]))
+    def test_be_list_twice(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_model_2")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["e 2", "q"]))
+    def test_be_list_report_excel(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_report_excel")
+        book = load_workbook(Files.be_list_excel)
+        sheet = book["STree"]
+        self.check_excel_sheet(sheet, "excel")
+
+    @patch(
+        "benchmark.Results.get_input", side_effect=iter(["e 2", "e 1", "q"])
+    )
+    def test_be_list_report_excel_twice(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["-m", "STree"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_report_excel_2")
+        book = load_workbook(Files.be_list_excel)
+        sheet = book["STree"]
+        self.check_excel_sheet(sheet, "excel")
+        sheet = book["STree2"]
+        self.check_excel_sheet(sheet, "excel2")
+
+    @patch("benchmark.Results.get_input", return_value="q")
+    def test_be_list_no_data(self, input_data):
         stdout, stderr = self.execute_script(
             "be_list", ["-m", "Wodt", "-s", "f1-macro"]
         )
         self.assertEqual(stderr.getvalue(), "")
         self.assertEqual(stdout.getvalue(), f"{NO_RESULTS}\n")
 
-    def test_be_list_nan(self):
+    @patch(
+        "benchmark.Results.get_input", side_effect=iter(["d 0", "y", "", "q"])
+    )
+    # @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
+    def test_be_list_delete(self, input_data):
+        def copy_files(source_folder, target_folder, file_name):
+            source = os.path.join(source_folder, file_name)
+            target = os.path.join(target_folder, file_name)
+            shutil.copyfile(source, target)
+
+        file_name = (
+            "results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:"
+            "35_0.json"
+        )
+        # move nan result from hidden to results
+        copy_files(Folders.hidden_results, Folders.results, file_name)
+        try:
+            # list and delete result
+            stdout, stderr = self.execute_script("be_list", "")
+            self.assertEqual(stderr.getvalue(), "")
+            self.check_output_file(stdout, "be_list_delete")
+        except Exception:
+            # delete the result copied if be_list couldn't
+            os.unlink(os.path.join(Folders.results, file_name))
+            self.fail("test_be_list_delete() should not raise exception")
+
+    @patch(
+        "benchmark.Results.get_input", side_effect=iter(["h 0", "y", "", "q"])
+    )
+    def test_be_list_hide(self, input_data):
         def swap_files(source_folder, target_folder, file_name):
             source = os.path.join(source_folder, file_name)
             target = os.path.join(target_folder, file_name)
@@ -32,19 +106,38 @@ class BeListTest(TestBase):
         # move nan result from hidden to results
         swap_files(Folders.hidden_results, Folders.results, file_name)
         try:
-            # list and move nan result to hidden
+            # list and move nan result to hidden again
-            stdout, stderr = self.execute_script("be_list", ["--nan", "1"])
+            stdout, stderr = self.execute_script("be_list", "")
             self.assertEqual(stderr.getvalue(), "")
-            self.check_output_file(stdout, "be_list_nan")
+            self.check_output_file(stdout, "be_list_hide")
         except Exception:
-            # move back nan result file if be_list couldn't
+            # delete the result copied if be_list couldn't
             swap_files(Folders.results, Folders.hidden_results, file_name)
-            self.fail("test_be_list_nan() should not raise exception")
+            self.fail("test_be_list_hide() should not raise exception")
 
-    def test_be_list_nan_no_nan(self):
+    @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "q"]))
-        stdout, stderr = self.execute_script("be_list", ["--nan", "1"])
+    def test_be_list_already_hidden(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "be_list_no_nan")
+        self.check_output_file(stdout, "be_list_already_hidden")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["h 0", "n", "q"]))
+    def test_be_list_dont_hide(self, input_data):
+        stdout, stderr = self.execute_script("be_list", "")
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_default")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
+    def test_be_list_hidden_nan(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden", "--nan"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_hidden_nan")
+
+    @patch("benchmark.Results.get_input", side_effect=iter(["q"]))
+    def test_be_list_hidden(self, input_data):
+        stdout, stderr = self.execute_script("be_list", ["--hidden"])
+        self.assertEqual(stderr.getvalue(), "")
+        self.check_output_file(stdout, "be_list_hidden")
 
     def test_be_no_env(self):
         path = os.getcwd()

benchmark/tests/scripts/Be_Main_test.py

@@ -1,4 +1,5 @@
 import os
+import json
 from io import StringIO
 from unittest.mock import patch
 from ...Results import Report
@@ -30,7 +31,7 @@ class BeMainTest(TestBase):
     def test_be_main_complete(self):
         stdout, _ = self.execute_script(
             "be_main",
-            ["-s", self.score, "-m", "STree", "--title", "test", "-r", "1"],
+            ["-s", self.score, "-m", "STree", "--title", "test", "-r"],
         )
         # keep the report name to delete it after
         report_name = stdout.getvalue().splitlines()[-1].split("in ")[1]
@@ -66,10 +67,8 @@ class BeMainTest(TestBase):
                 "STree",
                 "--title",
                 "test",
-                "-f",
+                "-b",
-                "1",
                 "-r",
-                "1",
             ],
         )
         # keep the report name to delete it after
@@ -79,6 +78,48 @@ class BeMainTest(TestBase):
             stdout, "be_main_best", [0, 2, 3, 5, 6, 7, 8, 9, 12, 13, 14]
         )
 
+    @patch("sys.stdout", new_callable=StringIO)
+    @patch("sys.stderr", new_callable=StringIO)
+    def test_be_main_incompatible_params(self, stdout, stderr):
+        m1 = (
+            "be_main: error: argument -b/--best_paramfile: not allowed with "
+            "argument -p/--hyperparameters"
+        )
+        m2 = (
+            "be_main: error: argument -g/--grid_paramfile: not allowed with "
+            "argument -p/--hyperparameters"
+        )
+        m3 = (
+            "be_main: error: argument -g/--grid_paramfile: not allowed with "
+            "argument -p/--hyperparameters"
+        )
+        m4 = m1
+        p0 = [
+            "-s",
+            self.score,
+            "-m",
+            "SVC",
+            "--title",
+            "test",
+        ]
+        pset = json.dumps(dict(C=17))
+        p1 = p0.copy()
+        p1.extend(["-p", pset, "-b"])
+        p2 = p0.copy()
+        p2.extend(["-p", pset, "-g"])
+        p3 = p0.copy()
+        p3.extend(["-p", pset, "-g", "-b"])
+        p4 = p0.copy()
+        p4.extend(["-b", "-g"])
+        parameters = [(p1, m1), (p2, m2), (p3, m3), (p4, m4)]
+        for parameter, message in parameters:
+            with self.assertRaises(SystemExit) as msg:
+                module = self.search_script("be_main")
+                module.main(parameter)
+            self.assertEqual(msg.exception.code, 2)
+            self.assertEqual(stderr.getvalue(), "")
+            self.assertRegexpMatches(stdout.getvalue(), message)
+
     def test_be_main_best_params_non_existent(self):
         model = "GBC"
         stdout, stderr = self.execute_script(
@@ -90,10 +131,8 @@ class BeMainTest(TestBase):
                 model,
                 "--title",
                 "test",
-                "-f",
+                "-b",
-                "1",
                 "-r",
-                "1",
             ],
         )
         self.assertEqual(stderr.getvalue(), "")
@@ -117,9 +156,7 @@ class BeMainTest(TestBase):
                 "--title",
                 "test",
                 "-g",
-                "1",
                 "-r",
-                "1",
             ],
         )
         self.assertEqual(stderr.getvalue(), "")
@@ -142,9 +179,7 @@ class BeMainTest(TestBase):
                 "--title",
                 "test",
                 "-g",
-                "1",
                 "-r",
-                "1",
             ],
         )
         # keep the report name to delete it after

benchmark/tests/scripts/Be_Print_Strees_test.py

@@ -18,7 +18,7 @@ class BePrintStrees(TestBase):
         for name in self.datasets:
             stdout, _ = self.execute_script(
                 "be_print_strees",
-                ["-d", name, "-q", "1"],
+                ["-d", name, "-q"],
             )
             file_name = os.path.join(Folders.img, f"stree_{name}.png")
             self.files.append(file_name)
@@ -33,7 +33,7 @@ class BePrintStrees(TestBase):
         for name in self.datasets:
             stdout, _ = self.execute_script(
                 "be_print_strees",
-                ["-d", name, "-q", "1", "-c", "1"],
+                ["-d", name, "-q", "-c"],
             )
             file_name = os.path.join(Folders.img, f"stree_{name}.png")
             self.files.append(file_name)

benchmark/tests/scripts/Be_Report_test.py

@@ -1,7 +1,10 @@
 import os
 from openpyxl import load_workbook
+from io import StringIO
+from unittest.mock import patch
 from ...Utils import Folders, Files
 from ..TestBase import TestBase
+from ..._version import __version__
 
 
 class BeReportTest(TestBase):
@@ -14,6 +17,7 @@ class BeReportTest(TestBase):
             "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.xlsx",
         ]
         self.remove_files(files, Folders.results)
+        self.remove_files([Files.datasets_report_excel], os.getcwd())
         return super().tearDown()
 
     def test_be_report(self):
@@ -21,47 +25,68 @@ class BeReportTest(TestBase):
             "results",
             "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json",
         )
-        stdout, stderr = self.execute_script("be_report", ["-f", file_name])
+        stdout, stderr = self.execute_script("be_report", ["file", file_name])
         self.assertEqual(stderr.getvalue(), "")
         self.check_output_file(stdout, "report")
 
     def test_be_report_not_found(self):
-        stdout, stderr = self.execute_script("be_report", ["-f", "unknown"])
+        stdout, stderr = self.execute_script("be_report", ["file", "unknown"])
         self.assertEqual(stderr.getvalue(), "")
         self.assertEqual(stdout.getvalue(), "unknown does not exists!\n")
 
     def test_be_report_compare(self):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"
         stdout, stderr = self.execute_script(
-            "be_report", ["-f", file_name, "-c", "1"]
+            "be_report", ["file", file_name, "-c"]
         )
         self.assertEqual(stderr.getvalue(), "")
         self.check_output_file(stdout, "report_compared")
 
     def test_be_report_datatsets(self):
-        stdout, stderr = self.execute_script("be_report", [])
+        stdout, stderr = self.execute_script("be_report", ["datasets"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "report_datasets")
+        file_name = f"report_datasets{self.ext}"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        output_text = stdout.getvalue().splitlines()
+        for line, index in zip(expected.splitlines(), range(len(expected))):
+            if self.benchmark_version in line:
+                # replace benchmark version
+                line = self.replace_benchmark_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])
 
     def test_be_report_datasets_excel(self):
-        stdout, stderr = self.execute_script("be_report", ["-x", "1"])
+        stdout, stderr = self.execute_script("be_report", ["datasets", "-x"])
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "report_datasets")
+        file_name = f"report_datasets{self.ext}"
+        with open(os.path.join(self.test_files, file_name)) as f:
+            expected = f.read()
+        output_text = stdout.getvalue().splitlines()
+        for line, index in zip(expected.splitlines(), range(len(expected))):
+            if self.benchmark_version in line:
+                # replace benchmark version
+                line = self.replace_benchmark_version(line, output_text, index)
+            self.assertEqual(line, output_text[index])
         file_name = os.path.join(os.getcwd(), Files.datasets_report_excel)
         book = load_workbook(file_name)
         sheet = book["Datasets"]
-        self.check_excel_sheet(sheet, "exreport_excel_Datasets")
+        self.check_excel_sheet(
+            sheet,
+            "exreport_excel_Datasets",
+            replace=self.benchmark_version,
+            with_this=__version__,
+        )
 
     def test_be_report_best(self):
         stdout, stderr = self.execute_script(
-            "be_report", ["-s", "accuracy", "-m", "STree", "-b", "1"]
+            "be_report", ["best", "-s", "accuracy", "-m", "STree"]
         )
         self.assertEqual(stderr.getvalue(), "")
         self.check_output_file(stdout, "report_best")
 
     def test_be_report_grid(self):
         stdout, stderr = self.execute_script(
-            "be_report", ["-s", "accuracy", "-m", "STree", "-g", "1"]
+            "be_report", ["grid", "-s", "accuracy", "-m", "STree"]
         )
         self.assertEqual(stderr.getvalue(), "")
         file_name = "report_grid.test"
@@ -75,19 +100,24 @@ class BeReportTest(TestBase):
                 line = self.replace_STree_version(line, output_text, index)
             self.assertEqual(line, output_text[index])
 
-    def test_be_report_best_both(self):
+    @patch("sys.stderr", new_callable=StringIO)
-        stdout, stderr = self.execute_script(
+    def test_be_report_unknown_subcommand(self, stderr):
-            "be_report",
+        with self.assertRaises(SystemExit) as msg:
-            ["-s", "accuracy", "-m", "STree", "-b", "1", "-g", "1"],
+            module = self.search_script("be_report")
-        )
+            module.main(["unknown"])
+        self.assertEqual(msg.exception.code, 2)
+        self.check_output_file(stderr, "report_unknown_subcommand")
+
+    def test_be_report_without_subcommand(self):
+        stdout, stderr = self.execute_script("be_report", "")
         self.assertEqual(stderr.getvalue(), "")
-        self.check_output_file(stdout, "report_best")
+        self.check_output_file(stdout, "report_without_subcommand")
 
     def test_be_report_excel_compared(self):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"
         stdout, stderr = self.execute_script(
             "be_report",
-            ["-f", file_name, "-x", "1", "-c", "1"],
+            ["file", file_name, "-x", "-c"],
         )
         file_name = os.path.join(
             Folders.results, file_name.replace(".json", ".xlsx")
@@ -102,7 +132,7 @@ class BeReportTest(TestBase):
         file_name = "results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json"
         stdout, stderr = self.execute_script(
             "be_report",
-            ["-f", file_name, "-x", "1"],
+            ["file", file_name, "-x"],
         )
         file_name = os.path.join(
             Folders.results, file_name.replace(".json", ".xlsx")
@@ -117,7 +147,7 @@ class BeReportTest(TestBase):
         file_name = "results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json"
         stdout, stderr = self.execute_script(
             "be_report",
-            ["-f", file_name, "-q", "1"],
+            ["file", file_name, "-q"],
         )
         file_name = os.path.join(
             Folders.results, file_name.replace(".json", ".sql")

benchmark/tests/test_files/be_init_project.test

@@ -0,0 +1,10 @@
+Creating folder test_project
+Creating folder test_project/results
+Creating folder test_project/hidden_results
+Creating folder test_project/exreport
+Creating folder test_project/exreport/exreport_output
+Creating folder test_project/img
+Done!
+Please, edit .env file with your settings and add a datasets folder
+with an all.txt file with the datasets you want to use.
+In that folder you have to include all the datasets you'll use.

benchmark/tests/test_files/be_init_project_no_arguments.test

@@ -0,0 +1,2 @@
+usage: be_init_project [-h] project_name
+be_init_project: error: the following arguments are required: project_name

benchmark/tests/test_files/be_list_already_hidden.test

@@ -0,0 +1,5 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default
+Already hidden

benchmark/tests/test_files/be_list_default.test

@@ -0,0 +1,7 @@
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_delete.test

@@ -0,0 +1,16 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= ============================================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json            0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json   0.03627   0.076 Test default paramters with RandomForest
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json     0.03790   1.143 default B
+  4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json          0.04158   0.943 default A
+  5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json          0.04544   0.173 With gridsearched hyperparameters
+Deleting results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_hidden.test

@@ -0,0 +1,4 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default

benchmark/tests/test_files/be_list_hidden_nan.test

@@ -0,0 +1,3 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= =======================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters

benchmark/tests/test_files/be_list_hide.test

@@ -0,0 +1,16 @@
+ #  Date       File                                                             Score    Time(h) Title
+=== ========== ================================================================ ======== ======= ============================================
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json            0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  2 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json   0.03627   0.076 Test default paramters with RandomForest
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json     0.03790   1.143 default B
+  4 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json          0.04158   0.943 default A
+  5 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json          0.04544   0.173 With gridsearched hyperparameters
+Hiding results/results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json
+ #  Date       File                                                            Score    Time(h) Title
+=== ========== =============================================================== ======== ======= ============================================
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_model.test

@@ -0,0 +1,5 @@
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_model_2.test

@@ -0,0 +1,10 @@
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_model_invalid.test

@@ -0,0 +1,6 @@
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters
+Invalid option. Try again!

benchmark/tests/test_files/be_list_nan.test

@@ -1,13 +0,0 @@
-Date       File                                                             Score    Time(h) Title
-========== ================================================================ ======== ======= ============================================
-2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json            0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json   0.03627   0.076 Test default paramters with RandomForest
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json     0.03790   1.143 default B
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json          0.04158   0.943 default A
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json          0.04544   0.173 With gridsearched hyperparameters
-
-****************************** Results with nan moved to hidden ******************************
-Date       File                                                             Score    Time(h) Title
-========== ================================================================ ======== ======= =======================
-2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters

benchmark/tests/test_files/be_list_no_nan.test

@@ -1,7 +0,0 @@
-Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/be_list_report.test

@@ -0,0 +1,20 @@
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters
+*************************************************************************************************************************
+* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-11-01 19:17:07             *
+* default B                                                                                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
+* Execution took 4115.04 seconds,   1.14 hours, on macbook-pro                                                          *
+* Score is accuracy                                                                                                     *
+*************************************************************************************************************************
+
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
+balance-scale                     625     4   3   18.78    9.88    5.90 0.970000±0.0020   0.233304±0.0481 {'max_features': 'auto', 'splitter': 'mutual'} 
+balloons                           16     4   2    4.72    2.86    2.78 0.556667±0.2941   0.021352±0.0058 {'max_features': 'auto', 'splitter': 'mutual'} 
+*************************************************************************************************************************
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0379                                                         *
+*************************************************************************************************************************

benchmark/tests/test_files/be_list_report_excel.test

@@ -0,0 +1,7 @@
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters
+Added results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json to some_results.xlsx
+Generated file: some_results.xlsx

benchmark/tests/test_files/be_list_report_excel_2.test

@@ -0,0 +1,8 @@
+ #  Date       File                                                          Score    Time(h) Title
+=== ========== ============================================================= ======== ======= =================================
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters
+Added results/results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json to some_results.xlsx
+Added results/results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json to some_results.xlsx
+Generated file: some_results.xlsx

benchmark/tests/test_files/be_main_best.test

@@ -1,16 +1,16 @@
-************************************************************************************************************************
+*************************************************************************************************************************
-* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25            *
+* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:15:25             *
-* test                                                                                                                 *
+* test                                                                                                                  *
-* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
-* Execution took    0.80 seconds,   0.00 hours, on iMac27                                                              *
+* Execution took    0.80 seconds,   0.00 hours, on iMac27                                                               *
-* Score is accuracy                                                                                                    *
+* Score is accuracy                                                                                                     *
-************************************************************************************************************************
+*************************************************************************************************************************
 
-Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time             Hyperparameters 
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
-============================== ====== ===== === ======= ======= ======= =============== ================ =============== 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
-balance-scale                      625     4   3   23.32   12.16    6.44 0.840160±0.0304  0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} 
+balance-scale                      625     4   3   23.32   12.16    6.44 0.840160±0.0304   0.013745±0.0019 {'splitter': 'best', 'max_features': 'auto'} 
-balloons                            16     4   2    3.00    2.00    2.00 0.860000±0.2850  0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
+balloons                            16     4   2    3.00    2.00    2.00 0.860000±0.2850   0.000388±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
-************************************************************************************************************************
+*************************************************************************************************************************
-* accuracy compared to STree_default (liblinear-ovr) .:  0.0422                                                        *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0422                                                         *
-************************************************************************************************************************
+*************************************************************************************************************************
 Results in results/results_accuracy_STree_iMac27_2022-05-09_00:15:25_0.json

benchmark/tests/test_files/be_main_complete.test

@@ -1,16 +1,16 @@
-************************************************************************************************************************
+*************************************************************************************************************************
-* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43            *
+* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 20:14:43             *
-* test                                                                                                                 *
+* test                                                                                                                  *
-* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
-* Execution took    0.48 seconds,   0.00 hours, on iMac27                                                              *
+* Execution took    0.48 seconds,   0.00 hours, on iMac27                                                               *
-* Score is accuracy                                                                                                    *
+* Score is accuracy                                                                                                     *
-************************************************************************************************************************
+*************************************************************************************************************************
 
-Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time             Hyperparameters 
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
-============================== ====== ===== === ======= ======= ======= =============== ================ =============== 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
-balance-scale                     625     4   3   17.36    9.18    6.18 0.908480±0.0247  0.007388±0.0013 {}              
+balance-scale                     625     4   3   17.36    9.18    6.18 0.908480±0.0247   0.007388±0.0013 {}              
-balloons                           16     4   2    4.64    2.82    2.66 0.663333±0.3009  0.000664±0.0002 {}              
+balloons                           16     4   2    4.64    2.82    2.66 0.663333±0.3009   0.000664±0.0002 {}              
-************************************************************************************************************************
+*************************************************************************************************************************
-* accuracy compared to STree_default (liblinear-ovr) .:  0.0390                                                        *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0390                                                         *
-************************************************************************************************************************
+*************************************************************************************************************************
 Results in results/results_accuracy_STree_iMac27_2022-05-08_20:14:43_0.json

benchmark/tests/test_files/be_main_dataset.test

@@ -1,15 +1,15 @@
-************************************************************************************************************************
+*************************************************************************************************************************
-* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28            *
+* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-08 19:38:28             *
-* test                                                                                                                 *
+* Test with only one dataset                                                                                            *
-* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
-* Execution took    0.06 seconds,   0.00 hours, on iMac27                                                              *
+* Execution took    0.06 seconds,   0.00 hours, on iMac27                                                               *
-* Score is accuracy                                                                                                    *
+* Score is accuracy                                                                                                     *
-************************************************************************************************************************
+*************************************************************************************************************************
 
-Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time             Hyperparameters 
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
-============================== ====== ===== === ======= ======= ======= =============== ================ =============== 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
-balloons                           16     4   2    4.64    2.82    2.66 0.663333±0.3009  0.000671±0.0001 {}              
+balloons                           16     4   2    4.64    2.82    2.66 0.663333±0.3009   0.000671±0.0001 {}              
-************************************************************************************************************************
+*************************************************************************************************************************
-* accuracy compared to STree_default (liblinear-ovr) .:  0.0165                                                        *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0165                                                         *
-************************************************************************************************************************
+*************************************************************************************************************************
 Partial result file removed: results/results_accuracy_STree_iMac27_2022-05-08_19:38:28_0.json

benchmark/tests/test_files/be_main_grid.test

@@ -1,16 +1,16 @@
-************************************************************************************************************************
+*************************************************************************************************************************
-* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06            *
+* STree ver. 1.2.4 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2022-05-09 00:21:06             *
-* test                                                                                                                 *
+* test                                                                                                                  *
-* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
-* Execution took    0.89 seconds,   0.00 hours, on iMac27                                                              *
+* Execution took    0.89 seconds,   0.00 hours, on iMac27                                                               *
-* Score is accuracy                                                                                                    *
+* Score is accuracy                                                                                                     *
-************************************************************************************************************************
+*************************************************************************************************************************
 
-Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time             Hyperparameters 
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
-============================== ====== ===== === ======= ======= ======= =============== ================ =============== 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
-balance-scale                     625     4   3   26.12   13.56    7.94 0.910720±0.0249  0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} 
+balance-scale                     625     4   3   26.12   13.56    7.94 0.910720±0.0249   0.015852±0.0027 {'C': 1.0, 'kernel': 'liblinear', 'multiclass_strategy': 'ovr'} 
-balloons                           16     4   2    4.64    2.82    2.66 0.663333±0.3009  0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} 
+balloons                           16     4   2    4.64    2.82    2.66 0.663333±0.3009   0.000640±0.0001 {'C': 1.0, 'kernel': 'linear', 'multiclass_strategy': 'ovr'} 
-************************************************************************************************************************
+*************************************************************************************************************************
-* accuracy compared to STree_default (liblinear-ovr) .:  0.0391                                                        *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0391                                                         *
-************************************************************************************************************************
+*************************************************************************************************************************
 Results in results/results_accuracy_STree_iMac27_2022-05-09_00:21:06_0.json

benchmark/tests/test_files/be_summary_list_model.test

@@ -26,10 +26,10 @@
 * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json                      *
 *                                                                               *
 *********************************************************************************
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest

benchmark/tests/test_files/be_summary_list_score.test

@@ -26,10 +26,10 @@
 * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json                      *
 *                                                                               *
 *********************************************************************************
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest

benchmark/tests/test_files/be_summary_list_score_all.test

@@ -26,13 +26,13 @@
 * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json                      *
 *                                                                               *
 *********************************************************************************
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
 ** No results found **
 ** No results found **
 ** No results found **

benchmark/tests/test_files/be_summary_list_score_model.test

@@ -26,10 +26,10 @@
 * results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json                      *
 *                                                                               *
 *********************************************************************************
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest

benchmark/tests/test_files/excel2.test

@@ -0,0 +1,48 @@
+1;1;" STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-10-27 09:40:40"
+2;1;" default A"
+3;1;" Score is accuracy"
+3;2;" Execution time"
+3;5;"3,395.01 s"
+3;7;" "
+3;8;"Platform"
+3;9;"iMac27"
+3;10;"Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1]"
+4;5;"  0.94 h"
+4;10;"Stratified: False"
+6;1;"Dataset"
+6;2;"Samples"
+6;3;"Features"
+6;4;"Classes"
+6;5;"Nodes"
+6;6;"Leaves"
+6;7;"Depth"
+6;8;"Score"
+6;9;"Score Std."
+6;10;"Time"
+6;11;"Time Std."
+6;12;"Hyperparameters"
+7;1;"balance-scale"
+7;2;"625"
+7;3;"4"
+7;4;"3"
+7;5;"11.08"
+7;6;"5.9"
+7;7;"5.9"
+7;8;"0.98"
+7;9;"0.001"
+7;10;"0.2852065515518188"
+7;11;"0.06031593282605064"
+7;12;"{'splitter': 'best', 'max_features': 'auto'}"
+8;1;"balloons"
+8;2;"16"
+8;3;"4"
+8;4;"2"
+8;5;"4.12"
+8;6;"2.56"
+8;7;"2.56"
+8;8;"0.695"
+8;9;"0.2756860130252853"
+8;10;"0.02120100021362305"
+8;11;"0.003526023309468471"
+8;12;"{'splitter': 'best', 'max_features': 'auto'}"
+10;1;"** accuracy compared to STree_default (liblinear-ovr) .:  0.0416"

benchmark/tests/test_files/report.test

@@ -1,15 +1,15 @@
-************************************************************************************************************************
+*************************************************************************************************************************
-* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07            *
+* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07             *
-* With gridsearched hyperparameters                                                                                    *
+* With gridsearched hyperparameters                                                                                     *
-* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
-* Execution took  624.25 seconds,   0.17 hours, on iMac27                                                              *
+* Execution took  624.25 seconds,   0.17 hours, on iMac27                                                               *
-* Score is accuracy                                                                                                    *
+* Score is accuracy                                                                                                     *
-************************************************************************************************************************
+*************************************************************************************************************************
 
-Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time             Hyperparameters 
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
-============================== ====== ===== === ======= ======= ======= =============== ================ =============== 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
-balance-scale                     625     4   3    7.00    4.00    3.00 0.970560±0.0150  0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
+balance-scale                     625     4   3    7.00    4.00    3.00 0.970560±0.0150   0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
-balloons                           16     4   2    3.00    2.00    2.00 0.860000±0.2850  0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
+balloons                           16     4   2    3.00    2.00    2.00 0.860000±0.2850   0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
-************************************************************************************************************************
+*************************************************************************************************************************
-* accuracy compared to STree_default (liblinear-ovr) .:  0.0454                                                        *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0454                                                         *
-************************************************************************************************************************
+*************************************************************************************************************************

benchmark/tests/test_files/report_compared.test

@@ -1,16 +1,16 @@
-************************************************************************************************************************
+*************************************************************************************************************************
-* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07            *
+* STree ver. 1.2.3 Python ver. 3.11x with 5 Folds cross validation and 10 random seeds. 2021-09-30 11:42:07             *
-* With gridsearched hyperparameters                                                                                    *
+* With gridsearched hyperparameters                                                                                     *
-* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                             *
+* Random seeds: [57, 31, 1714, 17, 23, 79, 83, 97, 7, 1] Stratified: False                                              *
-* Execution took  624.25 seconds,   0.17 hours, on iMac27                                                              *
+* Execution took  624.25 seconds,   0.17 hours, on iMac27                                                               *
-* Score is accuracy                                                                                                    *
+* Score is accuracy                                                                                                     *
-************************************************************************************************************************
+*************************************************************************************************************************
 
-Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time             Hyperparameters 
+Dataset                        Sampl. Feat. Cls Nodes   Leaves  Depth   Score           Time              Hyperparameters 
-============================== ====== ===== === ======= ======= ======= =============== ================ =============== 
+============================== ====== ===== === ======= ======= ======= =============== ================= =============== 
-balance-scale                     625     4   3    7.00    4.00    3.00 0.970560±0.0150  0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
+balance-scale                     625     4   3    7.00    4.00    3.00 0.970560±0.0150   0.014049±0.0020 {'C': 10000.0, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
-balloons                           16     4   2    3.00    2.00    2.00 0.860000±0.2850✔ 0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
+balloons                           16     4   2    3.00    2.00    2.00 0.860000±0.2850✔  0.000854±0.0000 {'C': 7, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': 10000.0, 'multiclass_strategy': 'ovr'} 
-************************************************************************************************************************
+*************************************************************************************************************************
-* ✔ Equal to best .....:  1                                                                                            *
+* ✔ Equal to best .....:  1                                                                                             *
-* accuracy compared to STree_default (liblinear-ovr) .:  0.0454                                                        *
+* accuracy compared to STree_default (liblinear-ovr) .:  0.0454                                                         *
-************************************************************************************************************************
+*************************************************************************************************************************

benchmark/tests/test_files/report_unknown_subcommand.test

@@ -0,0 +1,2 @@
+usage: be_report [-h] {best,grid,file,datasets} ...
+be_report: error: argument subcommand: invalid choice: 'unknown' (choose from 'best', 'grid', 'file', 'datasets')

benchmark/tests/test_files/report_without_subcommand.test

@@ -0,0 +1,12 @@
+usage: be_report [-h] {best,grid,file,datasets} ...
+
+positional arguments:
+  {best,grid,file,datasets}
+                        help for subcommand
+    best                Report best results obtained by any model/score. See be_build_best
+    grid                Report grid results obtained by any model/score. See be_build_grid
+    file                Report file results
+    datasets            Report datasets information
+
+options:
+  -h, --help            show this help message and exit

benchmark/tests/test_files/summary_list_hidden.test

@@ -1,4 +1,4 @@
-Date       File                                                             Score    Time(h) Title
+ #  Date       File                                                             Score    Time(h) Title
-========== ================================================================ ======== ======= =======================
+=== ========== ================================================================ ======== ======= =======================
-2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
+  0 2022-05-04 results_accuracy_XGBoost_MacBookpro16_2022-05-04_11:00:35_0.json      nan   3.091 Default hyperparameters
-2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default
+  1 2021-11-01 results_accuracy_STree_iMac27_2021-11-01_23:55:16_0.json          0.97446   0.098 default

benchmark/tests/test_files/summary_list_model.test

@@ -1,5 +1,5 @@
-Date       File                                                          Score    Time(h) Title
+ #  Date       File                                                          Score    Time(h) Title
-========== ============================================================= ======== ======= =================================
+=== ========== ============================================================= ======== ======= =================================
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
+  0 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json  0.03790   1.143 default B
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
+  1 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json       0.04158   0.943 default A
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters
+  2 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json       0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/summary_list_n.test

@@ -1,5 +1,5 @@
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B

benchmark/tests/test_files/summary_list_score.test

@@ -1,7 +1,7 @@
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  0 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  1 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  2 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  3 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+  4 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters

benchmark/tests/test_files/summary_show_top.test

@@ -1,7 +1,7 @@
-Date       File                                                            Score    Time(h) Title
+ #  Date       File                                                            Score    Time(h) Title
-========== =============================================================== ======== ======= ============================================
+=== ========== =============================================================== ======== ======= ============================================
-2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
+  0 2021-09-30 results_accuracy_STree_iMac27_2021-09-30_11:42:07_0.json         0.04544   0.173 With gridsearched hyperparameters
-2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
+  1 2022-04-20 results_accuracy_ODTE_Galgo_2022-04-20_10:52:20_0.json           0.04341   6.275 Gridsearched hyperparams v022.1b random_init
-2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
+  2 2021-10-27 results_accuracy_STree_iMac27_2021-10-27_09:40:40_0.json         0.04158   0.943 default A
-2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
+  3 2021-11-01 results_accuracy_STree_macbook-pro_2021-11-01_19:17:07_0.json    0.03790   1.143 default B
-2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest
+  4 2022-01-14 results_accuracy_RandomForest_iMac27_2022-01-14_12:39:30_0.json  0.03627   0.076 Test default paramters with RandomForest

setup.py

@@ -38,6 +38,7 @@ def script_names():
         "print_strees",
         "report",
         "summary",
+        "init_project",
     ]
     result = []
     for script in scripts:
@@ -49,20 +50,17 @@ setuptools.setup(
     name="benchmark",
     version=get_data("version", "_version.py"),
     license=get_data("license"),
-    description="Oblique decision tree with svm nodes",
+    description="Benchmark of models with different datasets",
     long_description=readme(),
     long_description_content_type="text/markdown",
     packages=setuptools.find_packages(),
     url="https://github.com/Doctorado-ML/benchmark",
     author=get_data("author"),
     author_email=get_data("author_email"),
-    keywords="scikit-learn oblique-classifier oblique-decision-tree decision-\
+    keywords="scikit-learn benchmark",
-    tree svm svc",
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: " + get_data("license"),
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Natural Language :: English",
         "Topic :: Scientific/Engineering :: Artificial Intelligence",