rmontanana/Platform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add model selection to b_best to filter results

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2024-03-09 20:19:27 +01:00
parent 7e4ee0a9a9
commit 4c847fc3f6
3 changed files with 20 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/best/BestResults.cc
View File

@@ -42,6 +42,9 @@ namespace platform {
for (auto const& item : data.at("results")) { for (auto const& item : data.at("results")) {
bool update = true; bool update = true;
auto datasetName = item.at("dataset").get<std::string>(); auto datasetName = item.at("dataset").get<std::string>();
if (dataset != "any" && dataset != datasetName) {
continue;
}
if (bests.contains(datasetName)) { if (bests.contains(datasetName)) {
if (item.at("score").get<double>() < bests[datasetName].at(0).get<double>()) { if (item.at("score").get<double>() < bests[datasetName].at(0).get<double>()) {
update = false; update = false;
@@ -122,8 +125,8 @@ namespace platform {
std::vector<std::string> BestResults::getDatasets(json table) std::vector<std::string> BestResults::getDatasets(json table)
{ {
std::vector<std::string> datasets; std::vector<std::string> datasets;
for (const auto& dataset : table.items()) { for (const auto& dataset_ : table.items()) {
datasets.push_back(dataset.key()); datasets.push_back(dataset_.key());
} }
maxDatasetName = (*max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size(); maxDatasetName = (*max_element(datasets.begin(), datasets.end(), [](const std::string& a, const std::string& b) { return a.size() < b.size(); })).size();
maxDatasetName = std::max(7, maxDatasetName); maxDatasetName = std::max(7, maxDatasetName);
@@ -232,16 +235,16 @@ namespace platform {
totals[model] = 0.0; totals[model] = 0.0;
} }
auto datasets = getDatasets(table.begin().value()); auto datasets = getDatasets(table.begin().value());
for (auto const& dataset : datasets) { for (auto const& dataset_ : datasets) {
auto color = odd ? Colors::BLUE() : Colors::CYAN(); auto color = odd ? Colors::BLUE() : Colors::CYAN();
std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " "; std::cout << color << std::setw(3) << std::fixed << std::right << i++ << " ";
std::cout << std::setw(maxDatasetName) << std::left << dataset << " "; std::cout << std::setw(maxDatasetName) << std::left << dataset_ << " ";
double maxValue = 0; double maxValue = 0;
// Find out the max value for this dataset // Find out the max value for this dataset
for (const auto& model : models) { for (const auto& model : models) {
double value; double value;
try { try {
value = table[model].at(dataset).at(0).get<double>(); value = table[model].at(dataset_).at(0).get<double>();
} }
catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) { catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) {
value = -1.0; value = -1.0;
@@ -255,7 +258,7 @@ namespace platform {
std::string efectiveColor = color; std::string efectiveColor = color;
double value; double value;
try { try {
value = table[model].at(dataset).at(0).get<double>(); value = table[model].at(dataset_).at(0).get<double>();
} }
catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) { catch (nlohmann::json_abi_v3_11_3::detail::out_of_range err) {
value = -1.0; value = -1.0;
@@ -331,9 +334,9 @@ namespace platform {
double min = 2000; double min = 2000;
// Find out the control model // Find out the control model
auto totals = std::vector<double>(models.size(), 0.0); auto totals = std::vector<double>(models.size(), 0.0);
for (const auto& dataset : datasets) { for (const auto& dataset_ : datasets) {
for (int i = 0; i < models.size(); ++i) { for (int i = 0; i < models.size(); ++i) {
totals[i] += ranksModels[dataset][models[i]]; totals[i] += ranksModels[dataset_][models[i]];
} }
} }
for (int i = 0; i < models.size(); ++i) { for (int i = 0; i < models.size(); ++i) {

5
src/best/BestResults.h
View File

@@ -6,8 +6,8 @@ using json = nlohmann::json;
namespace platform { namespace platform {
class BestResults { class BestResults {
public: public:
explicit BestResults(const std::string& path, const std::string& score, const std::string& model, bool friedman, double significance = 0.05) explicit BestResults(const std::string& path, const std::string& score, const std::string& model, const std::string& dataset, bool friedman, double significance = 0.05)
: path(path), score(score), model(model), friedman(friedman), significance(significance) : path(path), score(score), model(model), dataset(dataset), friedman(friedman), significance(significance)
{ {
} }
std::string build(); std::string build();
@@ -27,6 +27,7 @@ namespace platform {
std::string path; std::string path;
std::string score; std::string score;
std::string model; std::string model;
std::string dataset;
bool friedman; bool friedman;
double significance; double significance;
int maxModelName = 0; int maxModelName = 0;

10
src/best/b_best.cc
View File

@@ -8,6 +8,7 @@
void manageArguments(argparse::ArgumentParser& program) void manageArguments(argparse::ArgumentParser& program)
{ {
program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)"); program.add_argument("-m", "--model").default_value("").help("Filter results of the selected model) (any for all models)");
program.add_argument("-d", "--dataset").default_value("any").help("Filter results of the selected model) (any for all datasets)");
program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied"); program.add_argument("-s", "--score").default_value("accuracy").help("Filter results of the score name supplied");
program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true); program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true);
program.add_argument("--excel").help("Output to excel").default_value(false).implicit_value(true); program.add_argument("--excel").help("Output to excel").default_value(false).implicit_value(true);
@@ -31,12 +32,13 @@ int main(int argc, char** argv)
{ {
argparse::ArgumentParser program("b_best", { platform_project_version.begin(), platform_project_version.end() }); argparse::ArgumentParser program("b_best", { platform_project_version.begin(), platform_project_version.end() });
manageArguments(program); manageArguments(program);
std::string model, score; std::string model, dataset, score;
bool build, report, friedman, excel; bool build, report, friedman, excel;
double level; double level;
try { try {
program.parse_args(argc, argv); program.parse_args(argc, argv);
model = program.get<std::string>("model"); model = program.get<std::string>("model");
dataset = program.get<std::string>("dataset");
score = program.get<std::string>("score"); score = program.get<std::string>("score");
friedman = program.get<bool>("friedman"); friedman = program.get<bool>("friedman");
excel = program.get<bool>("excel"); excel = program.get<bool>("excel");
@@ -44,8 +46,8 @@ int main(int argc, char** argv)
if (model == "" || score == "") { if (model == "" || score == "") {
throw std::runtime_error("Model and score name must be supplied"); throw std::runtime_error("Model and score name must be supplied");
} }
if (friedman && model != "any") { if (friedman && (model != "any" || dataset != "any")) {
std::cerr << "Friedman test can only be used with all models" << std::endl; std::cerr << "Friedman test can only be used with all models and all the datasets" << std::endl;
std::cerr << program; std::cerr << program;
exit(1); exit(1);
} }
@@ -56,7 +58,7 @@ int main(int argc, char** argv)
exit(1); exit(1);
} }
// Generate report // Generate report
auto results = platform::BestResults(platform::Paths::results(), score, model, friedman, level); auto results = platform::BestResults(platform::Paths::results(), score, model, dataset, friedman, level);
if (model == "any") { if (model == "any") {
results.buildAll(); results.buildAll();
results.reportAll(excel); results.reportAll(excel);