rmontanana/BayesNet
rmontanana/BayesNet
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 3 Wiki Activity

Add friedman hyperparameter

Browse Source
This commit is contained in:
Ricardo Montañana Gómez 2023-09-26 11:26:59 +02:00
parent f0d0abe891
commit cab8e14b2d
Signed by: rmontanana
GPG Key ID: 46064262FD9A7ADE
3 changed files with 23 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/Platform/BestResults.cc
View File

@ -237,11 +237,9 @@ namespace platform {
cout << "Can't make the Friedman test with less than 3 models and/or less than 3 datasets." << endl;
return;
}
cout << Colors::BLUE() << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << endl;
cout << "N datasets: " << nDatasets << endl;
cout << "N models: " << nModels << endl;
cout << "Significance: " << significance << endl;
cout << "Nº Ranks: " << ranks.size() << endl;
cout << Colors::BLUE() << endl;
cout << "*************************************************************************************" << endl;
cout << "Friedman test: H0: 'There is no significant differences between all the classifiers.'" << endl;
for (const auto& rank : ranks) {
sum += rank.second;
}
@ -250,22 +248,21 @@ namespace platform {
for (const auto& rank : ranks) {
sumSquared += rank.second * rank.second;
}
cout << "Sum Squared: " << sumSquared << endl;
cout << "Degrees of freedom: " << degreesOfFreedom << endl;
double friedman = 12.0 / (nModels * nDatasets * (nModels + 1)) * sumSquared - 3 * nDatasets * (nModels + 1);
cout << "Friedman statistic: " << friedman << endl;
double friedmanQ = 12.0 / (nModels * nDatasets * (nModels + 1)) * sumSquared - 3 * nDatasets * (nModels + 1);
cout << "Friedman statistic: " << friedmanQ << endl;
// Calculate the critical value
boost::math::chi_squared chiSquared(degreesOfFreedom);
long double p_value = (long double)1.0 - cdf(chiSquared, friedman);
long double p_value = (long double)1.0 - cdf(chiSquared, friedmanQ);
double criticalValue = quantile(chiSquared, 1 - significance);
std::cout << "Critical Chi-Square Value for df=" << degreesOfFreedom
std::cout << "Critical Chi-Square Value for df=" << fixed << (int)degreesOfFreedom
<< " and alpha=" << significance << ": " << criticalValue << std::endl;
cout << "p-value: " << scientific << p_value << endl;
if (friedman > criticalValue) {
if (friedmanQ > criticalValue) {
cout << Colors::MAGENTA() << "The null hypothesis H0 is rejected." << endl;
} else {
cout << Colors::GREEN() << "The null hypothesis H0 is accepted." << endl;
}
cout << Colors::BLUE() << "*************************************************************************************" << endl;
}
void BestResults::printTableResults(set<string> models, json table)
{
@ -371,7 +368,9 @@ namespace platform {
cout << efectiveColor << setw(12) << setprecision(9) << fixed << (double)ranksTotal[model] / (double)origin.size() << " ";
}
cout << endl;
friedmanTest(models.size(), table.begin().value().size(), ranksTotal, 0.05);
if (friedman) {
friedmanTest(models.size(), table.begin().value().size(), ranksTotal, 0.05);
}
}
void BestResults::reportAll()
{

3
src/Platform/BestResults.h
View File

@ -8,7 +8,7 @@ using json = nlohmann::json;
namespace platform {
class BestResults {
public:
explicit BestResults(const string& path, const string& score, const string& model) : path(path), score(score), model(model) {}
explicit BestResults(const string& path, const string& score, const string& model, bool friedman) : path(path), score(score), model(model), friedman(friedman) {}
string build();
void reportSingle();
void reportAll();
@ -23,6 +23,7 @@ namespace platform {
string path;
string score;
string model;
bool friedman;
};
}
#endif //BESTRESULTS_H

10
src/Platform/best.cc
View File

@ -13,12 +13,14 @@ argparse::ArgumentParser manageArguments(int argc, char** argv)
program.add_argument("-s", "--score").default_value("").help("Filter results of the score name supplied");
program.add_argument("--build").help("build best score results file").default_value(false).implicit_value(true);
program.add_argument("--report").help("report of best score results file").default_value(false).implicit_value(true);
program.add_argument("--friedman").help("Friedman test").default_value(false).implicit_value(true);
try {
program.parse_args(argc, argv);
auto model = program.get<string>("model");
auto score = program.get<string>("score");
auto build = program.get<bool>("build");
auto report = program.get<bool>("report");
auto friedman = program.get<bool>("friedman");
if (model == "" || score == "") {
throw runtime_error("Model and score name must be supplied");
}
@ -38,12 +40,18 @@ int main(int argc, char** argv)
auto score = program.get<string>("score");
auto build = program.get<bool>("build");
auto report = program.get<bool>("report");
auto friedman = program.get<bool>("friedman");
if (friedman && model != "any") {
cerr << "Friedman test can only be used with all models" << endl;
cerr << program;
exit(1);
}
if (!report && !build) {
cerr << "Either build, report or both, have to be selected to do anything!" << endl;
cerr << program;
exit(1);
}
auto results = platform::BestResults(platform::Paths::results(), score, model);
auto results = platform::BestResults(platform::Paths::results(), score, model, friedman);
if (build) {
if (model == "any") {
results.buildAll();