rmontanana/Platform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add results to b_list

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2024-03-10 18:02:03 +01:00
parent 05d05e25c2
commit cd9ff89b52
9 changed files with 184 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/CMakeLists.txt
View File

@@ -19,34 +19,49 @@ include_directories(
set(best_sources b_best.cpp BestResults.cpp Statistics.cpp BestResultsExcel.cpp)
list(TRANSFORM best_sources PREPEND best/)
add_executable(
b_best ${best_sources} main/Result.cpp
reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp common/Datasets.cpp common/Dataset.cpp main/Models.cpp)
b_best ${best_sources}
common/Datasets.cpp common/Dataset.cpp
main/Result.cpp main/Models.cpp
reports/ReportExcel.cpp reports/ReportBase.cpp reports/ExcelFile.cpp
)
target_link_libraries(b_best Boost::boost "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy "${XLSXWRITER_LIB}")
# b_grid
set(grid_sources b_grid.cpp GridSearch.cpp GridData.cpp)
list(TRANSFORM grid_sources PREPEND grid/)
add_executable(b_grid ${grid_sources} main/HyperParameters.cpp main/Models.cpp common/Datasets.cpp common/Dataset.cpp)
add_executable(b_grid ${grid_sources}
common/Datasets.cpp common/Dataset.cpp
main/HyperParameters.cpp main/Models.cpp
)
target_link_libraries(b_grid ${MPI_CXX_LIBRARIES} "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy)
# b_list
set(list_sources b_list.cpp DatasetsExcel.cpp)
set(list_sources b_list.cpp DatasetsExcel.cpp ResultsDataset.cpp)
list(TRANSFORM list_sources PREPEND list/)
add_executable(b_list ${list_sources} common/Datasets.cpp common/Dataset.cpp reports/ReportExcel.cpp reports/ExcelFile.cpp reports/ReportBase.cpp)
target_link_libraries(b_list "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)
add_executable(b_list ${list_sources}
common/Datasets.cpp common/Dataset.cpp
main/Models.cpp
reports/ReportExcel.cpp reports/ExcelFile.cpp reports/ReportBase.cpp main/Result.cpp
)
target_link_libraries(b_list "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy "${XLSXWRITER_LIB}")
# b_main
set(main_sources b_main.cpp Experiment.cpp Models.cpp HyperParameters.cpp)
list(TRANSFORM main_sources PREPEND main/)
add_executable(b_main ${main_sources} common/Datasets.cpp common/Dataset.cpp reports/ReportConsole.cpp reports/ReportBase.cpp main/Result.cpp)
add_executable(b_main ${main_sources}
common/Datasets.cpp common/Dataset.cpp
main/Result.cpp
reports/ReportConsole.cpp reports/ReportBase.cpp
)
target_link_libraries(b_main "${PyClassifiers}" "${BayesNet}" ArffFiles mdlp ${Python3_LIBRARIES} "${TORCH_LIBRARIES}" ${LIBTORCH_PYTHON} Boost::python Boost::numpy)
# b_manage
set(manage_sources b_manage.cpp ManageResults.cpp CommandParser.cpp ResultsManager.cpp)
list(TRANSFORM manage_sources PREPEND manage/)
add_executable(
b_manage ${manage_sources} main/Result.cpp
reports/ReportConsole.cpp reports/ReportExcel.cpp reports/ReportExcelCompared.cpp reports/ReportBase.cpp reports/ExcelFile.cpp
b_manage ${manage_sources}
common/Datasets.cpp common/Dataset.cpp
main/Result.cpp
reports/ReportConsole.cpp reports/ReportExcel.cpp reports/ReportExcelCompared.cpp reports/ReportBase.cpp reports/ExcelFile.cpp
)
target_link_libraries(b_manage "${TORCH_LIBRARIES}" "${XLSXWRITER_LIB}" ArffFiles mdlp)

4
src/best/BestResults.cpp
View File

@@ -170,10 +170,9 @@ namespace platform {
std::cout << Colors::GREEN() << " # " << std::setw(maxDatasetName + 1) << std::left << "Dataset" << "Score " << std::setw(maxFileName) << "File" << " Hyperparameters" << std::endl;
std::cout << "=== " << std::string(maxDatasetName, '=') << " =========== " << std::string(maxFileName, '=') << " " << std::string(maxHyper, '=') << std::endl;
auto i = 0;
bool odd = true;
double total = 0;
for (auto const& item : data.items()) {
auto color = odd ? Colors::BLUE() : Colors::CYAN();
auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN();
double value = item.value().at(0).get<double>();
std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " ";
std::cout << std::setw(maxDatasetName) << std::left << item.key() << " ";
@@ -182,7 +181,6 @@ namespace platform {
std::cout << item.value().at(1) << " ";
std::cout << std::endl;
total += value;
odd = !odd;
}
std::cout << Colors::GREEN() << "=== " << std::string(maxDatasetName, '=') << " ===========" << std::endl;
std::cout << Colors::GREEN() << " Total" << std::string(maxDatasetName - 5, '.') << " " << std::setw(11) << std::setprecision(8) << std::fixed << total << std::endl;

54
src/list/ResultsDataset.cpp Normal file
View File

@@ -0,0 +1,54 @@
#include <algorithm>
#include "common/Paths.h"
#include "ResultsDataset.h"
namespace platform {
ResultsDataset::ResultsDataset(const std::string& dataset, const std::string& model, const std::string& score) :
path(Paths::results()), dataset(dataset), model(model), scoreName(score), maxModel(0), maxFile(0), maxHyper(15), maxResult(0)
{
}
void ResultsDataset::load()
{
using std::filesystem::directory_iterator;
for (const auto& file : directory_iterator(path)) {
auto filename = file.path().filename().string();
if (filename.find(".json") != std::string::npos && filename.find("results_") == 0) {
auto result = Result();
result.load(path, filename);
if (model != "any" && result.getModel() != model)
continue;
auto data = result.getData()["results"];
for (auto const& item : data) {
if (item["dataset"] == dataset) {
auto hyper_length = item["hyperparameters"].dump().size();
if (hyper_length > maxHyper)
maxHyper = hyper_length;
if (item["score"].get<double>() > maxResult)
maxResult = item["score"].get<double>();
files.push_back(result);
break;
}
}
}
}
maxModel = std::max(size_t(5), (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getModel().size() < b.getModel().size(); })).getModel().size());
maxFile = std::max(size_t(4), (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getFilename().size() < b.getFilename().size(); })).getFilename().size());
}
int ResultsDataset::size() const
{
return files.size();
}
void ResultsDataset::sortModel()
{
sort(files.begin(), files.end(), [](const Result& a, const Result& b) {
if (a.getModel() == b.getModel()) {
return a.getDate() > b.getDate();
}
return a.getModel() < b.getModel();
});
}
bool ResultsDataset::empty() const
{
return files.empty();
}
}

34
src/list/ResultsDataset.h Normal file
View File

@@ -0,0 +1,34 @@
#pragma once
#include <vector>
#include <string>
#include <nlohmann/json.hpp>
#include "main/Result.h"
namespace platform {
using json = nlohmann::json;
class ResultsDataset {
public:
ResultsDataset(const std::string& dataset, const std::string& model, const std::string& score);
void load(); // Loads the list of results
void sortModel();
int maxModelSize() const { return maxModel; };
int maxFileSize() const { return maxFile; };
int maxHyperSize() const { return maxHyper; };
double maxResultScore() const { return maxResult; };
int size() const;
bool empty() const;
std::vector<Result>::iterator begin() { return files.begin(); };
std::vector<Result>::iterator end() { return files.end(); };
Result& at(int index) { return files.at(index); };
private:
std::string path;
std::string dataset;
std::string model;
std::string scoreName;
int maxModel;
int maxFile;
int maxHyper;
double maxResult;
std::vector<Result> files;
};
};

62
src/list/b_list.cpp
View File

@@ -3,10 +3,13 @@
#include <map>
#include <argparse/argparse.hpp>
#include <nlohmann/json.hpp>
#include "main/Models.h"
#include "main/modelRegister.h"
#include "common/Paths.h"
#include "common/Colors.h"
#include "common/Datasets.h"
#include "DatasetsExcel.h"
#include "ResultsDataset.h"
#include "config.h"
const int BALANCE_LENGTH = 75;
@@ -32,7 +35,7 @@ std::string outputBalance(const std::string& balance)
void list_datasets(argparse::ArgumentParser& program)
{
auto datasets = platform::Datasets(false, platform::Paths::datasets());
auto excel = program.get<bool>("--excel");
auto excel = program.get<bool>("excel");
locale mylocale(std::cout.getloc(), new separated);
locale::global(mylocale);
std::cout.imbue(mylocale);
@@ -74,10 +77,44 @@ void list_datasets(argparse::ArgumentParser& program)
void list_results(argparse::ArgumentParser& program)
{
std::cout << "Results" << std::endl;
auto dataset = program.get<string>("--dataset");
auto score = program.get<string>("--score");
auto dataset = program.get<string>("dataset");
auto score = program.get<string>("score");
auto model = program.get<string>("model");
auto results = platform::ResultsDataset(dataset, model, score);
results.load();
results.sortModel();
if (results.empty()) {
std::cerr << Colors::RED() << "No results found for dataset " << dataset << " and model " << model << Colors::RESET() << std::endl;
exit(1);
}
//
// List data
//
int maxModel = results.maxModelSize();
int maxFileName = results.maxFileSize();
int maxHyper = results.maxHyperSize();
double maxResult = results.maxResultScore();
std::cout << Colors::GREEN() << "Results for dataset " << dataset << std::endl;
std::cout << "There are " << results.size() << " results" << std::endl;
std::cout << Colors::GREEN() << " # " << std::setw(maxModel + 1) << std::left << "Model" << "Date Score " << std::setw(maxFileName) << "File" << " Hyperparameters" << std::endl;
std::cout << "=== " << std::string(maxModel, '=') << " ========== =========== " << std::string(maxFileName, '=') << " " << std::string(maxHyper, '=') << std::endl;
auto i = 0;
for (const auto& result : results) {
auto data = result.getData();
for (const auto& item : data["results"]) {
if (item["dataset"] == dataset) {
auto color = (i % 2) ? Colors::BLUE() : Colors::CYAN();
color = item["score"].get<double>() == maxResult ? Colors::RED() : color;
std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " ";
std::cout << std::setw(maxModel) << std::left << result.getModel() << " ";
std::cout << color << result.getDate() << " ";
std::cout << std::setw(11) << std::setprecision(9) << std::fixed << item["score"].get<double>() << " ";
std::cout << std::setw(maxFileName) << result.getFilename() << " ";
std::cout << item["hyperparameters"].dump() << std::endl;
break;
}
}
}
}
int main(int argc, char** argv)
@@ -110,7 +147,20 @@ int main(int argc, char** argv)
throw std::runtime_error("Dataset must be one of " + datasets.toString());
}
);
program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied");
results_command.add_argument("-m", "--model")
.help("Model to use: " + platform::Models::instance()->toString() + " or any")
.default_value("any")
.action([](const std::string& value) {
std::vector<std::string> valid(platform::Models::instance()->getNames());
valid.push_back("any");
static const std::vector<std::string> choices = valid;
if (find(choices.begin(), choices.end(), value) != choices.end()) {
return value;
}
throw std::runtime_error("Model must be one of " + platform::Models::instance()->toString() + " or any");
}
);
results_command.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied");
// Add subparsers
program.add_subparser(datasets_command);
program.add_subparser(results_command);

1
src/main/Result.h
View File

@@ -27,6 +27,7 @@ namespace platform {
std::string getModel() const { return data["model"].get<std::string>(); };
std::string getScoreName() const { return data["score_name"].get<std::string>(); };
bool isComplete() const { return complete; };
json getData() const { return data; }
// Setters
void setTitle(const std::string& title) { data["title"] = title; };
void setLanguage(const std::string& language) { data["language"] = language; };

2
src/manage/ManageResults.cpp
View File

@@ -14,6 +14,8 @@ namespace platform {
ManageResults::ManageResults(int numFiles, const std::string& model, const std::string& score, bool complete, bool partial, bool compare) :
numFiles{ numFiles }, complete{ complete }, partial{ partial }, compare{ compare }, results(ResultsManager(model, score, complete, partial))
{
results.load();
results.sortDate();
indexList = true;
openExcel = false;
workbook = NULL;

18
src/manage/ResultsManager.cpp
View File

@@ -4,14 +4,8 @@
namespace platform {
ResultsManager::ResultsManager(const std::string& model, const std::string& score, bool complete, bool partial) :
path(Paths::results()), model(model), scoreName(score), complete(complete), partial(partial)
path(Paths::results()), model(model), scoreName(score), complete(complete), partial(partial), maxModel(0)
{
load();
if (!files.empty()) {
maxModel = (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getModel().size() < b.getModel().size(); })).getModel().size();
} else {
maxModel = 0;
}
}
void ResultsManager::load()
{
@@ -28,6 +22,7 @@ namespace platform {
files.push_back(result);
}
}
maxModel = std::max(size_t(5), (*max_element(files.begin(), files.end(), [](const Result& a, const Result& b) { return a.getModel().size() < b.getModel().size(); })).getModel().size());
}
void ResultsManager::hideResult(int index, const std::string& pathHidden)
{
@@ -48,12 +43,18 @@ namespace platform {
void ResultsManager::sortDate()
{
sort(files.begin(), files.end(), [](const Result& a, const Result& b) {
if (a.getDate() == b.getDate()) {
return a.getModel() < b.getModel();
}
return a.getDate() > b.getDate();
});
}
void ResultsManager::sortModel()
{
sort(files.begin(), files.end(), [](const Result& a, const Result& b) {
if (a.getModel() == b.getModel()) {
return a.getDate() > b.getDate();
}
return a.getModel() > b.getModel();
});
}
@@ -66,6 +67,9 @@ namespace platform {
void ResultsManager::sortScore()
{
sort(files.begin(), files.end(), [](const Result& a, const Result& b) {
if (a.getScore() == b.getScore()) {
return a.getDate() > b.getDate();
}
return a.getScore() > b.getScore();
});
}

3
src/manage/ResultsManager.h
View File

@@ -1,6 +1,5 @@
#pragma once
#include <map>
#include <vector>
#include <string>
#include <nlohmann/json.hpp>
@@ -10,6 +9,7 @@ namespace platform {
class ResultsManager {
public:
ResultsManager(const std::string& model, const std::string& score, bool complete, bool partial);
void load(); // Loads the list of results
void sortDate();
void sortScore();
void sortModel();
@@ -30,6 +30,5 @@ namespace platform {
bool partial;
int maxModel;
std::vector<Result> files;
void load(); // Loads the list of results
};
};