rmontanana/ArffFiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

Merge pull request 'string_features' (#1) from string_features into main

Browse Source 
Reviewed-on: #1
This commit is contained in:
Ricardo Montañana Gómez
2024-06-13 08:20:54 +00:00
parent dbefa02d9c cf32b9ae58
commit a5316928d4
3 changed files with 48979 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
ArffFiles.hpp
View File

@@ -7,10 +7,12 @@
#include <sstream>
#include <fstream>
#include <cctype> // std::isdigit
#include <algorithm> // std::all_of
#include <algorithm> // std::all_of std::transform
#include <iostream> // TODO remove
class ArffFiles {
const std::string VERSION = "1.0.0";
const std::string VERSION = "1.1.0";
public:
ArffFiles() = default;
void load(const std::string& fileName, bool classLast = true)
@@ -28,8 +30,9 @@ public:
attributes.erase(attributes.begin());
labelIndex = 0;
}
preprocessDataset(labelIndex);
generateDataset(labelIndex);
};
}
void load(const std::string& fileName, const std::string& name)
{
int labelIndex;
@@ -48,27 +51,67 @@ public:
if (!found) {
throw std::invalid_argument("Class name not found");
}
preprocessDataset(labelIndex);
generateDataset(labelIndex);
};
std::vector<std::string> getLines() const { return lines; };
unsigned long int getSize() const { return lines.size(); };
std::string getClassName() const { return className; };
std::string getClassType() const { return classType; };
std::vector<std::string> getLabels() const { return labels; }
}
std::vector<std::string> getLines() const { return lines; }
unsigned long int getSize() const { return lines.size(); }
std::string getClassName() const { return className; }
std::string getClassType() const { return classType; }
std::map<std::string, std::vector<std::string>> getStates() const { return states; }
std::vector<std::string> getLabels() const { return states.at(className); }
static std::string trim(const std::string& source)
{
std::string s(source);
s.erase(0, s.find_first_not_of(" '\n\r\t"));
s.erase(s.find_last_not_of(" '\n\r\t") + 1);
return s;
};
std::vector<std::vector<float>>& getX() { return X; };
}
std::vector<std::vector<float>>& getX() { return X; }
std::vector<int>& getY() { return y; }
std::map<std::string, bool> getNumericAttributes() const { return numeric_features; }
std::vector<std::pair<std::string, std::string>> getAttributes() const { return attributes; };
std::vector<int> factorize(const std::vector<std::string>& labels_t)
std::vector<std::string> split(const std::string& text, char delimiter)
{
std::vector<std::string> result;
std::stringstream ss(text);
std::string token;
while (std::getline(ss, token, delimiter)) {
result.push_back(trim(token));
}
return result;
}
std::string version() const { return VERSION; }
protected:
std::vector<std::string> lines;
std::map<std::string, bool> numeric_features;
std::vector<std::pair<std::string, std::string>> attributes;
std::string className;
std::string classType;
std::vector<std::vector<float>> X;
std::vector<std::vector<std::string>> Xs;
std::vector<int> y;
std::map<std::string, std::vector<std::string>> states;
private:
void preprocessDataset(int labelIndex)
{
//
// Learn the numeric features
//
numeric_features.clear();
for (const auto& attribute : attributes) {
auto feature = attribute.first;
if (feature == className)
continue;
auto values = attribute.second;
std::transform(values.begin(), values.end(), values.begin(), ::toupper);
numeric_features[feature] = values == "REAL" || values == "INTEGER" || values == "NUMERIC";
}
}
std::vector<int> factorize(const std::string feature, const std::vector<std::string>& labels_t)
{
std::vector<int> yy;
labels.clear();
states.at(feature).clear();
yy.reserve(labels_t.size());
std::map<std::string, int> labelMap;
int i = 0;
@@ -77,54 +120,46 @@ public:
labelMap[label] = i++;
bool allDigits = std::all_of(label.begin(), label.end(), ::isdigit);
if (allDigits)
labels.push_back("Class " + label);
states[feature].push_back("Class " + label);
else
labels.push_back(label);
states[feature].push_back(label);
}
yy.push_back(labelMap[label]);
}
return yy;
};
std::string version() const { return VERSION; };
protected:
std::vector<std::string> lines;
std::vector<std::pair<std::string, std::string>> attributes;
std::string className;
std::string classType;
std::vector<std::vector<float>> X;
std::vector<int> y;
std::vector<std::string> labels;
private:
}
void generateDataset(int labelIndex)
{
X = std::vector<std::vector<float>>(attributes.size(), std::vector<float>(lines.size()));
Xs = std::vector<std::vector<std::string>>(attributes.size(), std::vector<std::string>(lines.size()));
auto yy = std::vector<std::string>(lines.size(), "");
auto removeLines = std::vector<int>(); // Lines with missing values
for (size_t i = 0; i < lines.size(); i++) {
std::stringstream ss(lines[i]);
std::string value;
int pos = 0;
int xIndex = 0;
while (getline(ss, value, ',')) {
auto tokens = split(lines[i], ',');
for (const auto& token : tokens) {
if (pos++ == labelIndex) {
yy[i] = value;
yy[i] = token;
} else {
if (value == "?") {
X[xIndex++][i] = -1;
removeLines.push_back(i);
} else
X[xIndex++][i] = stof(value);
if (numeric_features[attributes[xIndex].first]) {
X[xIndex][i] = stof(token);
} else {
Xs[xIndex][i] = token;
}
xIndex++;
}
}
}
for (auto i : removeLines) {
yy.erase(yy.begin() + i);
for (auto& x : X) {
x.erase(x.begin() + i);
for (size_t i = 0; i < attributes.size(); i++) {
if (!numeric_features[attributes[i].first]) {
auto data = factorize(attributes[i].first, Xs[i]);
std::transform(data.begin(), data.end(), X[i].begin(), [](int x) { return float(x);});
}
}
y = factorize(yy);
};
y = factorize(className, yy);
}
void loadCommon(std::string fileName)
{
std::ifstream file(fileName);
@@ -152,12 +187,19 @@ private:
if (line[0] == '@') {
continue;
}
if (line.find("?", 0) != std::string::npos) {
// ignore lines with missing values
continue;
}
lines.push_back(line);
}
file.close();
for (const auto& attribute : attributes) {
states[attribute.first] = std::vector<std::string>();
}
if (attributes.empty())
throw std::invalid_argument("No attributes found");
};
}
};
#endif

41
tests/TestArffFiles.cc
View File

@@ -18,7 +18,7 @@ public:
TEST_CASE("Version Test", "[ArffFiles]")
{
ArffFiles arff;
REQUIRE(arff.version() == "1.0.0");
REQUIRE(arff.version() == "1.1.0");
}
TEST_CASE("Load Test", "[ArffFiles]")
{
@@ -65,14 +65,14 @@ TEST_CASE("Load Test", "[ArffFiles]")
TEST_CASE("Load with class name", "[ArffFiles]")
{
ArffFiles arff;
arff.load(Paths::datasets("glass"), "Type");
arff.load(Paths::datasets("glass"), std::string("Type"));
REQUIRE(arff.getClassName() == "Type");
REQUIRE(arff.getClassType() == "{ 'build wind float', 'build wind non-float', 'vehic wind float', 'vehic wind non-float', containers, tableware, headlamps}");
REQUIRE(arff.getLabels().size() == 6);
REQUIRE(arff.getLabels()[0] == "'build wind float'");
REQUIRE(arff.getLabels()[1] == "'vehic wind float'");
REQUIRE(arff.getLabels()[0] == "build wind float");
REQUIRE(arff.getLabels()[1] == "vehic wind float");
REQUIRE(arff.getLabels()[2] == "tableware");
REQUIRE(arff.getLabels()[3] == "'build wind non-float'");
REQUIRE(arff.getLabels()[3] == "build wind non-float");
REQUIRE(arff.getLabels()[4] == "headlamps");
REQUIRE(arff.getLabels()[5] == "containers");
REQUIRE(arff.getSize() == 214);
@@ -108,12 +108,41 @@ TEST_CASE("Load with class name as first attribute", "[ArffFiles]")
{1.86094, 1.89165, 1.93921, 1.71752},
{-0.207383, -0.193249, -0.239664, -0.218572} }
};
auto X = arff.getX();
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 4; ++j)
REQUIRE(arff.getX()[i][j] == Catch::Approx(expected[i][j]));
REQUIRE(X[i][j] == Catch::Approx(expected[i][j]));
}
auto expected_y = std::vector<int>{ 0, 0, 0, 0 };
for (int i = 120; i < 124; ++i)
REQUIRE(arff.getY()[i] == expected_y[i - 120]);
}
TEST_CASE("Adult dataset", "[ArffFiles]")
{
ArffFiles arff;
arff.load(Paths::datasets("adult"), std::string("class"));
REQUIRE(arff.getClassName() == "class");
REQUIRE(arff.getClassType() == "{ >50K, <=50K }");
REQUIRE(arff.getLabels().size() == 2);
REQUIRE(arff.getLabels()[0] == "<=50K");
REQUIRE(arff.getLabels()[1] == ">50K");
REQUIRE(arff.getSize() == 45222);
REQUIRE(arff.getLines().size() == 45222);
REQUIRE(arff.getLines()[0] == "25, Private, 226802, 11th, 7, Never-married, Machine-op-inspct, Own-child, Black, Male, 0, 0, 40, United-States, <=50K");
auto X = arff.getX();
REQUIRE(X[0][0] == 25);
REQUIRE(X[1][0] == 0);
REQUIRE(X[2][0] == 226802);
REQUIRE(X[3][0] == 0);
REQUIRE(X[4][0] == 7);
REQUIRE(X[5][0] == 0);
REQUIRE(X[6][0] == 0);
REQUIRE(X[7][0] == 0);
REQUIRE(X[8][0] == 0);
REQUIRE(X[9][0] == 0);
REQUIRE(X[10][0] == 0);
REQUIRE(X[11][0] == 0);
REQUIRE(X[12][0] == 40);
REQUIRE(X[13][0] == 0);
}

48861
tests/data/adult.arff Normal file
View File

File diff suppressed because it is too large Load Diff