diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 13df068..195144b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,34 +19,49 @@ include_directories( set(best_sources b_best.cpp BestResults.cpp Statistics.cpp BestResultsExcel.cpp) list(TRANSFORM best_sources PREPEND best/) add_executable( - b_best ${best_sources} main/Result.cpp - reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp common/Datasets.cpp common/Dataset.cpp main/Models.cpp) + b_best ${best_sources} + common/Datasets.cpp common/Dataset.cpp + main/Result.cpp main/Models.cpp + reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp +) target_link_libraries(b_best Boost::boost "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy "${XLSXWRITER_LIB}") # b_grid set(grid_sources b_grid.cpp GridSearch.cpp GridData.cpp) list(TRANSFORM grid_sources PREPEND grid/) -add_executable(b_grid ${grid_sources} main/HyperParameters.cpp main/Models.cpp common/Datasets.cpp common/Dataset.cpp) +add_executable(b_grid ${grid_sources} + common/Datasets.cpp common/Dataset.cpp + main/HyperParameters.cpp main/Models.cpp +) target_link_libraries(b_grid ${MPI_CXX_LIBRARIES} "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy) # b_list -set(list_sources b_list.cpp DatasetsExcel.cpp) +set(list_sources b_list.cpp DatasetsExcel.cpp ResultsDataset.cpp) list(TRANSFORM list_sources PREPEND list/) -add_executable(b_list ${list_sources} common/Datasets.cpp common/Dataset.cpp reports/ReportExcel.cpp reports/ExcelFile.cpp reports/ReportBase.cpp) -target_link_libraries(b_list "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp) +add_executable(b_list ${list_sources} + common/Datasets.cpp common/Dataset.cpp + main/Models.cpp + reports/ReportExcel.cpp reports/ExcelFile.cpp reports/ReportBase.cpp main/Result.cpp +) +target_link_libraries(b_list "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy "${XLSXWRITER_LIB}") # b_main set(main_sources b_main.cpp Experiment.cpp Models.cpp HyperParameters.cpp) list(TRANSFORM main_sources PREPEND main/) -add_executable(b_main ${main_sources} common/Datasets.cpp common/Dataset.cpp reports/ReportConsole.cpp reports/ReportBase.cpp main/Result.cpp) +add_executable(b_main ${main_sources} + common/Datasets.cpp common/Dataset.cpp + main/Result.cpp + reports/ReportConsole.cpp reports/ReportBase.cpp +) target_link_libraries(b_main "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy) # b_manage set(manage_sources b_manage.cpp ManageResults.cpp CommandParser.cpp ResultsManager.cpp) list(TRANSFORM manage_sources PREPEND manage/) add_executable( - b_manage ${manage_sources} main/Result.cpp - reports/ReportConsole.cpp reports/ReportExcel.cpp reports/ReportExcelCompared.cpp reports/ReportBase.cpp reports/ExcelFile.cpp + b_manage ${manage_sources} common/Datasets.cpp common/Dataset.cpp + main/Result.cpp + reports/ReportConsole.cpp reports/ReportExcel.cpp reports/ReportExcelCompared.cpp reports/ReportBase.cpp reports/ExcelFile.cpp ) target_link_libraries(b_manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp) diff --git a/src/best/BestResults.cpp b/src/best/BestResults.cpp index 6ea29c9..8bff53c 100644 --- a/src/best/BestResults.cpp +++ b/src/best/BestResults.cpp @@ -170,10 +170,9 @@ namespace platform { std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << "Dataset" << "Score " << std::setw(maxFileName) << "File" << " Hyperparameters" << std::endl; std::cout << "=== " << std::string(maxDatasetName, '=') << " =========== " << std::string(maxFileName, '=') << " " << std::string(maxHyper, '=') << std::endl; auto i = 0; - bool odd = true; double total = 0; for (auto const& item : data.items()) { - auto color = odd ? Colors::BLUE() : Colors::CYAN(); + auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN(); double value = item.value().at(0).get(); std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " "; std::cout << std::setw(maxDatasetName) << std::left << item.key() << " "; @@ -182,7 +181,6 @@ namespace platform { std::cout << item.value().at(1) << " "; std::cout << std::endl; total += value; - odd = !odd; } std::cout << Colors::GREEN() << "=== " << std::string(maxDatasetName, '=') << " ===========" << std::endl; std::cout << Colors::GREEN() << " Total" << std::string(maxDatasetName - 5, '.') << " " << std::setw(11) << std::setprecision(8) << std::fixed << total << std::endl; diff --git a/src/list/ResultsDataset.cpp b/src/list/ResultsDataset.cpp new file mode 100644 index 0000000..2cc285d --- /dev/null +++ b/src/list/ResultsDataset.cpp @@ -0,0 +1,54 @@ +#include +#include "common/Paths.h" +#include "ResultsDataset.h" + +namespace platform { + ResultsDataset::ResultsDataset(const std::string& dataset, const std::string& model, const std::string& score) : + path(Paths::results()), dataset(dataset), model(model), scoreName(score), maxModel(0), maxFile(0), maxHyper(15), maxResult(0) + { + } + void ResultsDataset::load() + { + using std::filesystem::directory_iterator; + for (const auto& file : directory_iterator(path)) { + auto filename = file.path().filename().string(); + if (filename.find(".json") != std::string::npos && filename.find("results_") == 0) { + auto result = Result(); + result.load(path, filename); + if (model != "any" && result.getModel() != model) + continue; + auto data = result.getData()["results"]; + for (auto const& item : data) { + if (item["dataset"] == dataset) { + auto hyper_length = item["hyperparameters"].dump().size(); + if (hyper_length > maxHyper) + maxHyper = hyper_length; + if (item["score"].get() > maxResult) + maxResult = item["score"].get(); + files.push_back(result); + break; + } + } + } + } + maxModel = std::max(size_t(5), (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getModel().size() < b.getModel().size(); })).getModel().size()); + maxFile = std::max(size_t(4), (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getFilename().size() < b.getFilename().size(); })).getFilename().size()); + } + int ResultsDataset::size() const + { + return files.size(); + } + void ResultsDataset::sortModel() + { + sort(files.begin(), files.end(), [](const Result& a, const Result& b) { + if (a.getModel() == b.getModel()) { + return a.getDate() > b.getDate(); + } + return a.getModel() < b.getModel(); + }); + } + bool ResultsDataset::empty() const + { + return files.empty(); + } +} \ No newline at end of file diff --git a/src/list/ResultsDataset.h b/src/list/ResultsDataset.h new file mode 100644 index 0000000..4ff542c --- /dev/null +++ b/src/list/ResultsDataset.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include +#include "main/Result.h" +namespace platform { + using json = nlohmann::json; + class ResultsDataset { + public: + ResultsDataset(const std::string& dataset, const std::string& model, const std::string& score); + void load(); // Loads the list of results + void sortModel(); + int maxModelSize() const { return maxModel; }; + int maxFileSize() const { return maxFile; }; + int maxHyperSize() const { return maxHyper; }; + double maxResultScore() const { return maxResult; }; + int size() const; + bool empty() const; + std::vector::iterator begin() { return files.begin(); }; + std::vector::iterator end() { return files.end(); }; + Result& at(int index) { return files.at(index); }; + private: + std::string path; + std::string dataset; + std::string model; + std::string scoreName; + int maxModel; + int maxFile; + int maxHyper; + double maxResult; + std::vector files; + }; +}; \ No newline at end of file diff --git a/src/list/b_list.cpp b/src/list/b_list.cpp index 7058afc..b40307d 100644 --- a/src/list/b_list.cpp +++ b/src/list/b_list.cpp @@ -3,10 +3,13 @@ #include #include #include +#include "main/Models.h" +#include "main/modelRegister.h" #include "common/Paths.h" #include "common/Colors.h" #include "common/Datasets.h" #include "DatasetsExcel.h" +#include "ResultsDataset.h" #include "config.h" const int BALANCE_LENGTH = 75; @@ -32,7 +35,7 @@ std::string outputBalance(const std::string& balance) void list_datasets(argparse::ArgumentParser& program) { auto datasets = platform::Datasets(false, platform::Paths::datasets()); - auto excel = program.get("--excel"); + auto excel = program.get("excel"); locale mylocale(std::cout.getloc(), new separated); locale::global(mylocale); std::cout.imbue(mylocale); @@ -74,10 +77,44 @@ void list_datasets(argparse::ArgumentParser& program) void list_results(argparse::ArgumentParser& program) { - std::cout << "Results" << std::endl; - auto dataset = program.get("--dataset"); - auto score = program.get("--score"); - + auto dataset = program.get("dataset"); + auto score = program.get("score"); + auto model = program.get("model"); + auto results = platform::ResultsDataset(dataset, model, score); + results.load(); + results.sortModel(); + if (results.empty()) { + std::cerr << Colors::RED() << "No results found for dataset " << dataset << " and model " << model << Colors::RESET() << std::endl; + exit(1); + } + // + // List data + // + int maxModel = results.maxModelSize(); + int maxFileName = results.maxFileSize(); + int maxHyper = results.maxHyperSize(); + double maxResult = results.maxResultScore(); + std::cout << Colors::GREEN() << "Results for dataset " << dataset << std::endl; + std::cout << "There are " << results.size() << " results" << std::endl; + std::cout << Colors::GREEN() << " # " << std::setw(maxModel + 1) << std::left << "Model" << "Date Score " << std::setw(maxFileName) << "File" << " Hyperparameters" << std::endl; + std::cout << "=== " << std::string(maxModel, '=') << " ========== =========== " << std::string(maxFileName, '=') << " " << std::string(maxHyper, '=') << std::endl; + auto i = 0; + for (const auto& result : results) { + auto data = result.getData(); + for (const auto& item : data["results"]) { + if (item["dataset"] == dataset) { + auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN(); + color = item["score"].get() == maxResult ? Colors::RED() : color; + std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " "; + std::cout << std::setw(maxModel) << std::left << result.getModel() << " "; + std::cout << color << result.getDate() << " "; + std::cout << std::setw(11) << std::setprecision(9) << std::fixed << item["score"].get() << " "; + std::cout << std::setw(maxFileName) << result.getFilename() << " "; + std::cout << item["hyperparameters"].dump() << std::endl; + break; + } + } + } } int main(int argc, char** argv) @@ -110,7 +147,20 @@ int main(int argc, char** argv) throw std::runtime_error("Dataset must be one of " + datasets.toString()); } ); - program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied"); + results_command.add_argument("-m", "--model") + .help("Model to use: " + platform::Models::instance()->toString() + " or any") + .default_value("any") + .action([](const std::string& value) { + std::vector valid(platform::Models::instance()->getNames()); + valid.push_back("any"); + static const std::vector choices = valid; + if (find(choices.begin(), choices.end(), value) != choices.end()) { + return value; + } + throw std::runtime_error("Model must be one of " + platform::Models::instance()->toString() + " or any"); + } + ); + results_command.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied"); // Add subparsers program.add_subparser(datasets_command); program.add_subparser(results_command); diff --git a/src/main/Result.h b/src/main/Result.h index 8e79cef..2756f7d 100644 --- a/src/main/Result.h +++ b/src/main/Result.h @@ -27,6 +27,7 @@ namespace platform { std::string getModel() const { return data["model"].get(); }; std::string getScoreName() const { return data["score_name"].get(); }; bool isComplete() const { return complete; }; + json getData() const { return data; } // Setters void setTitle(const std::string& title) { data["title"] = title; }; void setLanguage(const std::string& language) { data["language"] = language; }; diff --git a/src/manage/ManageResults.cpp b/src/manage/ManageResults.cpp index 973d626..8063a69 100644 --- a/src/manage/ManageResults.cpp +++ b/src/manage/ManageResults.cpp @@ -14,6 +14,8 @@ namespace platform { ManageResults::ManageResults(int numFiles, const std::string& model, const std::string& score, bool complete, bool partial, bool compare) : numFiles{ numFiles }, complete{ complete }, partial{ partial }, compare{ compare }, results(ResultsManager(model, score, complete, partial)) { + results.load(); + results.sortDate(); indexList = true; openExcel = false; workbook = NULL; diff --git a/src/manage/ResultsManager.cpp b/src/manage/ResultsManager.cpp index e73e99b..4222931 100644 --- a/src/manage/ResultsManager.cpp +++ b/src/manage/ResultsManager.cpp @@ -4,14 +4,8 @@ namespace platform { ResultsManager::ResultsManager(const std::string& model, const std::string& score, bool complete, bool partial) : - path(Paths::results()), model(model), scoreName(score), complete(complete), partial(partial) + path(Paths::results()), model(model), scoreName(score), complete(complete), partial(partial), maxModel(0) { - load(); - if (!files.empty()) { - maxModel = (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getModel().size() < b.getModel().size(); })).getModel().size(); - } else { - maxModel = 0; - } } void ResultsManager::load() { @@ -28,6 +22,7 @@ namespace platform { files.push_back(result); } } + maxModel = std::max(size_t(5), (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getModel().size() < b.getModel().size(); })).getModel().size()); } void ResultsManager::hideResult(int index, const std::string& pathHidden) { @@ -48,12 +43,18 @@ namespace platform { void ResultsManager::sortDate() { sort(files.begin(), files.end(), [](const Result& a, const Result& b) { + if (a.getDate() == b.getDate()) { + return a.getModel() < b.getModel(); + } return a.getDate() > b.getDate(); }); } void ResultsManager::sortModel() { sort(files.begin(), files.end(), [](const Result& a, const Result& b) { + if (a.getModel() == b.getModel()) { + return a.getDate() > b.getDate(); + } return a.getModel() > b.getModel(); }); } @@ -66,6 +67,9 @@ namespace platform { void ResultsManager::sortScore() { sort(files.begin(), files.end(), [](const Result& a, const Result& b) { + if (a.getScore() == b.getScore()) { + return a.getDate() > b.getDate(); + } return a.getScore() > b.getScore(); }); } diff --git a/src/manage/ResultsManager.h b/src/manage/ResultsManager.h index 2e7c55b..9b61b0c 100644 --- a/src/manage/ResultsManager.h +++ b/src/manage/ResultsManager.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -10,6 +9,7 @@ namespace platform { class ResultsManager { public: ResultsManager(const std::string& model, const std::string& score, bool complete, bool partial); + void load(); // Loads the list of results void sortDate(); void sortScore(); void sortModel(); @@ -30,6 +30,5 @@ namespace platform { bool partial; int maxModel; std::vector files; - void load(); // Loads the list of results }; }; \ No newline at end of file