rmontanana/Platform
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add discretiz algorithm management to b_main & Dataset

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2024-06-07 09:00:51 +02:00
parent 2202a81782
commit 5dd3deca1a
5 changed files with 28 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/commands/b_main.cpp
View File

@@ -111,7 +111,7 @@ int main(int argc, char** argv)
cerr << program;
exit(1);
}
auto datasets = platform::Datasets(discretize_dataset, platform::Paths::datasets());
auto datasets = platform::Datasets(false, platform::Paths::datasets());
if (datasets_file != "") {
ifstream catalog(datasets_file);
if (catalog.is_open()) {

15
src/common/Dataset.cpp
View File

@@ -2,6 +2,7 @@
#include <fstream>
#include "Dataset.h"
namespace platform {
const std::string message_dataset_not_loaded = "Dataset not loaded.";
Dataset::Dataset(const Dataset& dataset) :
path(dataset.path), name(dataset.name), className(dataset.className), n_samples(dataset.n_samples),
n_features(dataset.n_features), numericFeatures(dataset.numericFeatures), features(dataset.features),
@@ -23,7 +24,7 @@ namespace platform {
if (loaded) {
return features;
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
int Dataset::getNFeatures() const
@@ -31,7 +32,7 @@ namespace platform {
if (loaded) {
return n_features;
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
int Dataset::getNSamples() const
@@ -39,7 +40,7 @@ namespace platform {
if (loaded) {
return n_samples;
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
std::map<std::string, std::vector<int>> Dataset::getStates() const
@@ -47,7 +48,7 @@ namespace platform {
if (loaded) {
return states;
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
pair<std::vector<std::vector<float>>&, std::vector<int>&> Dataset::getVectors()
@@ -55,7 +56,7 @@ namespace platform {
if (loaded) {
return { Xv, yv };
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
pair<std::vector<std::vector<int>>&, std::vector<int>&> Dataset::getVectorsDiscretized()
@@ -63,7 +64,7 @@ namespace platform {
if (loaded) {
return { Xd, yv };
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
pair<torch::Tensor&, torch::Tensor&> Dataset::getTensors()
@@ -72,7 +73,7 @@ namespace platform {
buildTensors();
return { X, y };
} else {
throw std::invalid_argument("Dataset not loaded.");
throw std::invalid_argument(message_dataset_not_loaded);
}
}
void Dataset::load_csv()

10
src/common/Datasets.h
View File

@@ -4,7 +4,13 @@
namespace platform {
class Datasets {
public:
explicit Datasets(bool discretize, std::string sfileType) : discretize(discretize), sfileType(sfileType) { load(); };
explicit Datasets(bool discretize, std::string sfileType, std::string discretizer_algo = "none") : discretize(discretize), sfileType(sfileType), discretizer_algo(discretizer_algo)
{
if (discretizer_algo == "none" && discretize) {
throw std::runtime_error("Can't discretize without discretization algorithm");
}
load();
};
std::vector<std::string> getNames();
std::vector<std::string> getFeatures(const std::string& name) const;
int getNSamples(const std::string& name) const;
@@ -17,6 +23,7 @@ namespace platform {
std::pair<std::vector<std::vector<float>>&, std::vector<int>&> getVectors(const std::string& name);
std::pair<std::vector<std::vector<int>>&, std::vector<int>&> getVectorsDiscretized(const std::string& name);
std::pair<torch::Tensor&, torch::Tensor&> getTensors(const std::string& name);
std::tuple<torch::Tensor&, torch::Tensor&, torch::Tensor&, torch::Tensor&> getTrainTestTensors(const std::vector<int>& train_idx, const std::vector<int>& test_idx);
bool isDataset(const std::string& name) const;
void loadDataset(const std::string& name) const;
std::string toString() const;
@@ -24,6 +31,7 @@ namespace platform {
std::string path;
fileType_t fileType;
std::string sfileType;
std::string discretizer_algo;
std::map<std::string, std::unique_ptr<Dataset>> datasets;
bool discretize;
void load(); // Loads the list of datasets

3
src/grid/GridSearch.cpp
View File

@@ -373,7 +373,8 @@ namespace platform {
MPI_Bcast(msg, tasks_size + 1, MPI_CHAR, config_mpi.manager, MPI_COMM_WORLD);
tasks = json::parse(msg);
delete[] msg;
auto datasets = Datasets(config.discretize, Paths::datasets());
auto env = platform::DotEnv();
auto datasets = Datasets(config.discretize, Paths::datasets(), env.get("discretiz_algo"));
if (config_mpi.rank == config_mpi.manager) {
//
// 2a. Producer delivers the tasks to the consumers

25
src/main/Experiment.cpp
View File

@@ -117,20 +117,19 @@ namespace platform {
{
auto datasets = Datasets(false, Paths::datasets()); // Never discretize here
// Get dataset
auto [X, y] = datasets.getTensors(fileName);
auto states = datasets.getStates(fileName);
// -------------- auto [X, y] = datasets.getTensors(fileName);
// -------------- auto states = datasets.getStates(fileName);
auto features = datasets.getFeatures(fileName);
auto samples = datasets.getNSamples(fileName);
auto className = datasets.getClassName(fileName);
auto labels = datasets.getLabels(fileName);
int num_classes = states[className].size() == 0 ? labels.size() : states[className].size();
int num_classes = labels.size();
if (!quiet) {
std::cout << " " << setw(5) << samples << " " << setw(5) << features.size() << flush;
}
// Prepare Result
auto partial_result = PartialResult();
auto [values, counts] = at::_unique(y);
partial_result.setSamples(X.size(1)).setFeatures(X.size(0)).setClasses(values.size(0));
partial_result.setSamples(samples).setFeatures(features.size()).setClasses(num_classes);
partial_result.setHyperparameters(hyperparameters.get(fileName));
// Initialize results std::vectors
int nResults = nfolds * static_cast<int>(randomSeeds.size());
@@ -170,18 +169,10 @@ namespace platform {
// Split train - test dataset
train_timer.start();
auto [train, test] = fold->getFold(nfold);
auto train_t = torch::tensor(train);
auto test_t = torch::tensor(test);
auto X_train = X.index({ "...", train_t });
auto y_train = y.index({ train_t });
auto X_test = X.index({ "...", test_t });
auto y_test = y.index({ test_t });
if (discretized) {
// compute states too
// discretizer->fit(X_train, y_train);
// X_train = discretizer->transform(X_train);
// X_test = discretizer->transform(X_test);
}
auto [X_train, X_test, y_train, y_test] = datasets.getTrainTestTensors(fileName, train, test);
// Posibilidad de quitar todos los métodos de datasets y dejar un sólo de getDataset que devuelva
// una referencia al objeto dataset y trabajar directamente con él.
auto states = datasets.getStates(fileName);
if (generate_fold_files)
generate_files(fileName, discretized, stratified, seed, nfold, X_train, y_train, X_test, y_test, train, test);
if (!quiet)