From 577ccb9c5a0d62690eed4ee47cd7481fce359ac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ricardo=20Montan=CC=83ana?= <rmontanana@gmail.com>
Date: Wed, 1 Dec 2021 13:01:26 +0100
Subject: [PATCH] Put Sumary class in Results Add criterion summary and add
 different scores

---
 src/Results.py | 77 +++++++++++++++++++++++++++++++++++++++++++++++---
 src/Summary.py | 40 ++++----------------------
 src/Utils.py   |  7 +++++
 3 files changed, 86 insertions(+), 38 deletions(-)

diff --git a/src/Results.py b/src/Results.py
index 5efa6f9..a672af4 100644
--- a/src/Results.py
+++ b/src/Results.py
@@ -6,7 +6,7 @@ import subprocess
 import xlsxwriter
 from tqdm import tqdm
 from Experiments import Datasets, BestResults
-from Utils import Folders, Files, Symbols
+from Utils import Folders, Files, Symbols, BEST_ACCURACY_STREE
 
 
 class BaseReport(abc.ABC):
@@ -165,7 +165,7 @@ class Report(BaseReport):
                 )
         self.header_line(
             f" Accuracy compared to stree_default (liblinear-ovr) .: "
-            f"{accuracy/40.282203:7.4f}"
+            f"{accuracy/BEST_ACCURACY_STREE:7.4f}"
         )
         self.header_line("*")
 
@@ -231,7 +231,7 @@ class ReportBest(BaseReport):
                 )
         self.header_line(
             f" Scores compared to stree_default accuracy (liblinear-ovr) .: "
-            f"{accuracy/40.282203:7.4f}"
+            f"{accuracy/BEST_ACCURACY_STREE:7.4f}"
         )
         self.header_line("*")
 
@@ -354,7 +354,7 @@ class Excel(BaseReport):
                 self.row += 1
         message = (
             f"** Accuracy compared to stree_default (liblinear-ovr) .: "
-            f"{accuracy/40.282203:7.4f}"
+            f"{accuracy/BEST_ACCURACY_STREE:7.4f}"
         )
         bold = self.book.add_format({"bold": True, "font_size": 14})
         self.sheet.write(self.row + 1, 0, message, bold)
@@ -626,3 +626,72 @@ class Benchmark:
         footer()
 
         book.close()
+
+
+class StubReport(BaseReport):
+    def __init__(self, file_name):
+        super().__init__(file_name=file_name, best_file=False)
+
+    def print_line(self, line) -> None:
+        pass
+
+    def header(self) -> None:
+        pass
+
+    def footer(self, accuracy: float) -> None:
+        self.accuracy = accuracy
+
+
+class Summary:
+    def __init__(self) -> None:
+        self.results = Files().get_all_results()
+        self.data = []
+
+    def acquire(self) -> None:
+        """Get all results"""
+        for result in self.results:
+            (
+                score,
+                model,
+                platform,
+                date,
+                time,
+                stratified,
+            ) = Files().split_file_name(result)
+            report = StubReport(os.path.join(Folders.results, result))
+            report.report()
+            entry = dict(
+                score=score,
+                model=model,
+                platform=platform,
+                date=date,
+                time=time,
+                stratified=stratified,
+                file=result,
+                metric=report.accuracy / BEST_ACCURACY_STREE,
+            )
+            self.data.append(entry)
+
+    def list(self) -> None:
+        """Print the list of results"""
+        max_length = max(len(x["file"]) for x in self.data)
+        print(
+            "\n".join(
+                [
+                    f"{x['file']:{max_length}s} {x['metric']:7.3f}"
+                    for x in self.data
+                ]
+            )
+        )
+
+    def best_result(
+        self, criterion=None, value=None, score="accuracy"
+    ) -> dict:
+        # First filter the same score results (accuracy, f1, ...)
+        haystack = [x for x in self.data if x["score"] == score]
+        haystack = (
+            haystack
+            if criterion is None or value is None
+            else [x for x in haystack if x[criterion] == value]
+        )
+        return sorted(haystack, key=lambda x: x["metric"], reverse=True)[0]
diff --git a/src/Summary.py b/src/Summary.py
index 39c857a..cde0dd2 100644
--- a/src/Summary.py
+++ b/src/Summary.py
@@ -1,35 +1,7 @@
-import os
-from Utils import Folders, Files
-from Results import BaseReport
+from Results import Summary
 
-
-class StubReport(BaseReport):
-    def __init__(self, file_name):
-        super().__init__(file_name=file_name, best_file=False)
-
-    def print_line(self, line) -> None:
-        pass
-
-    def header(self) -> None:
-        pass
-
-    def footer(self, accuracy: float) -> None:
-        self.accuracy = accuracy
-
-
-class Summary:
-    def __init__(self) -> None:
-        self.results = Files().get_all_results()
-
-    def list(self) -> None:
-        """List all results"""
-        max_length = max([len(x) for x in self.results])
-        for result in self.results:
-            report = StubReport(os.path.join(Folders.results, result))
-            report.report()
-            print(f"{result:{max_length}s} {report.accuracy:7.3f}")
-        print("\n".join(self.results))
-
-
-if __name__ == "__main__":
-    Summary().list()
+summary = Summary()
+summary.acquire()
+print(summary.best_result())
+print(summary.best_result(criterion="model", value="ODTE"))
+summary.list()
diff --git a/src/Utils.py b/src/Utils.py
index d7a5494..fe45295 100644
--- a/src/Utils.py
+++ b/src/Utils.py
@@ -2,6 +2,8 @@ import os
 import subprocess
 import argparse
 
+BEST_ACCURACY_STREE = 40.282203
+
 
 class Folders:
     data = "data"
@@ -47,6 +49,11 @@ class Files:
             f"{stratified}.json"
         )
 
+    def split_file_name(self, name):
+        _, score, model, platform, date, time, stratified = name.split("_")
+        stratified = stratified.replace(self.report_ext, "")
+        return score, model, platform, date, time, stratified
+
     def results_suffixes(self, score="", model=""):
         suffix = self.report_ext
         if model == "" and score == "":