Complete adding string features

ArffFiles.hpp

@@ -7,7 +7,9 @@
 #include <sstream>
 #include <fstream>
 #include <cctype> // std::isdigit
-#include <algorithm> // std::all_of
+#include <algorithm> // std::all_of std::transform
+
+#include <iostream> // TODO remove
 
 class ArffFiles {
     const std::string VERSION = "1.1.0";
@@ -28,48 +30,8 @@ public:
             attributes.erase(attributes.begin());
             labelIndex = 0;
         }
+        preprocessDataset(labelIndex);
         generateDataset(labelIndex);
-    };
-    void preprocessDataset(int labelIndex)
-    {
-        //
-        // Learn the type of features
-        //
-        bool goodLine = false;
-        std::vector<std::string> tokens;
-        auto removeLines = std::vector<int>(); // Lines with missing values
-        int i = 0;
-        // Select a line with no missing values
-        while (!goodLine && i < lines.size()) {
-            tokens = split(lines[i], ',');
-            for (const auto& token : tokens) {
-                goodLine = true;
-                if (token == "?") {
-                    goodLine = false;
-                    removeLines.push_back(i);
-                    break;
-                }
-
-            }
-            i++;
-        }
-        // Remove lines in reverse order
-        std::sort(removeLines.begin(), removeLines.end(), std::greater<int>());
-        for (auto i : removeLines) {
-            lines.erase(lines.begin() + i);
-        }
-        numeric_features = std::vector<bool>(attributes.size(), true);
-        for (size_t i = 0; i < tokens.size(); i++) {
-            if (i == labelIndex) {
-                continue;
-            }
-            try {
-                stof(tokens[i]);
-            }
-            catch (std::invalid_argument& e) {
-                numeric_features[i] = false;
-            }
-        }
     }
     void load(const std::string& fileName, const std::string& name)
     {
@@ -91,12 +53,12 @@ public:
         }
         preprocessDataset(labelIndex);
         generateDataset(labelIndex);
-    };
+    }
-    std::vector<std::string> getLines() const { return lines; };
+    std::vector<std::string> getLines() const { return lines; }
-    unsigned long int getSize() const { return lines.size(); };
+    unsigned long int getSize() const { return lines.size(); }
-    std::string getClassName() const { return className; };
+    std::string getClassName() const { return className; }
-    std::string getClassType() const { return classType; };
+    std::string getClassType() const { return classType; }
-    std::map<std::string, std::vector<std::string>> getStates() const { return states; };
+    std::map<std::string, std::vector<std::string>> getStates() const { return states; }
     std::vector<std::string> getLabels() const { return states.at(className); }
     static std::string trim(const std::string& source)
     {
@@ -104,9 +66,10 @@ public:
         s.erase(0, s.find_first_not_of(" '\n\r\t"));
         s.erase(s.find_last_not_of(" '\n\r\t") + 1);
         return s;
-    };
+    }
-    std::vector<std::vector<float>>& getX() { return X; };
+    std::vector<std::vector<float>>& getX() { return X; }
     std::vector<int>& getY() { return y; }
+    std::vector<bool> getNumericAttributes() const { return numeric_features; }
     std::vector<std::pair<std::string, std::string>> getAttributes() const { return attributes; };
     std::vector<std::string> split(const std::string& text, char delimiter)
     {
@@ -118,7 +81,7 @@ public:
         }
         return result;
     }
-    std::string version() const { return VERSION; };
+    std::string version() const { return VERSION; }
 protected:
     std::vector<std::string> lines;
     std::vector<bool> numeric_features;
@@ -128,8 +91,23 @@ protected:
     std::vector<std::vector<float>> X;
     std::vector<std::vector<std::string>> Xs;
     std::vector<int> y;
-    std::map<std::string, std::vector<std:string>> states;
+    std::map<std::string, std::vector<std::string>> states;
 private:
+    void preprocessDataset(int labelIndex)
+    {
+        //
+        // Learn the numeric features
+        //
+        numeric_features = std::vector<bool>(attributes.size(), false);
+        for (size_t i = 0; i < attributes.size(); i++) {
+            if (i == labelIndex) {
+                continue;
+            }
+            std::string values = attributes.at(i).second;
+            std::transform(values.begin(), values.end(), values.begin(), ::toupper);
+            numeric_features[i] = values == "REAL" || values == "INTEGER" || values == "NUMERIC";
+        }
+    }
     std::vector<int> factorize(const std::string feature, const std::vector<std::string>& labels_t)
     {
         std::vector<int> yy;
@@ -149,13 +127,12 @@ private:
             yy.push_back(labelMap[label]);
         }
         return yy;
-    };
+    }
     void generateDataset(int labelIndex)
     {
         X = std::vector<std::vector<float>>(attributes.size(), std::vector<float>(lines.size()));
         Xs = std::vector<std::vector<std::string>>(attributes.size(), std::vector<std::string>(lines.size()));
         auto yy = std::vector<std::string>(lines.size(), "");
-        auto removeLines = std::vector<int>(); // Lines with missing values
         for (size_t i = 0; i < lines.size(); i++) {
             std::stringstream ss(lines[i]);
             std::string value;
@@ -166,17 +143,12 @@ private:
                 if (pos++ == labelIndex) {
                     yy[i] = token;
                 } else {
-                    if (token == "?") {
+                    if (numeric_features[xIndex]) {
-                        X[xIndex++][i] = -1;
+                        X[xIndex][i] = stof(token);
-                        removeLines.push_back(i);
                     } else {
-                        if (numeric_features[xIndex]) {
+                        Xs[xIndex][i] = token;
-                            X[xIndex][i] = stof(token);
-                        } else {
-                            Xs[xIndex][i] = token;
-                        }
-                        xIndex++;
                     }
+                    xIndex++;
                 }
             }
         }
@@ -184,33 +156,10 @@ private:
             if (!numeric_features[i]) {
                 auto data = factorize(attributes[i].first, Xs[i]);
                 std::transform(data.begin(), data.end(), X[i].begin(), [](int x) { return float(x);});
-            } else {
-                states[attributes[i].first] = std::vector<std::string>(Xs[i].begin(), Xs[i].end());
-            }
-        }
-        // Remove lines in reverse order
-        std::sort(removeLines.begin(), removeLines.end(), std::greater<int>());
-        for (auto i : removeLines) {
-            yy.erase(yy.begin() + i);
-            for (auto& x : X) {
-                try {
-                    x.erase(x.begin() + i);
-                }
-                catch (std::out_of_range& e) {
-                    continue;
-                }
-            }
-            for (auto& x : Xs) {
-                try {
-                    x.erase(x.begin() + i);
-                }
-                catch (std::out_of_range& e) {
-                    continue;
-                }
             }
         }
         y = factorize(className, yy);
-    };
+    }
     void loadCommon(std::string fileName)
     {
         std::ifstream file(fileName);
@@ -238,12 +187,19 @@ private:
             if (line[0] == '@') {
                 continue;
             }
+            if (line.find("?", 0) != std::string::npos) {
+                // ignore lines with missing values
+                continue;
+            }
             lines.push_back(line);
         }
         file.close();
+        for (const auto& attribute : attributes) {
+            states[attribute.first] = std::vector<std::string>();
+        }
         if (attributes.empty())
             throw std::invalid_argument("No attributes found");
-    };
+    }
 };
 
 #endif

tests/TestArffFiles.cc

@@ -18,7 +18,7 @@ public:
 TEST_CASE("Version Test", "[ArffFiles]")
 {
     ArffFiles arff;
-    REQUIRE(arff.version() == "1.0.0");
+    REQUIRE(arff.version() == "1.1.0");
 }
 TEST_CASE("Load Test", "[ArffFiles]")
 {
@@ -65,14 +65,14 @@ TEST_CASE("Load Test", "[ArffFiles]")
 TEST_CASE("Load with class name", "[ArffFiles]")
 {
     ArffFiles arff;
-    arff.load(Paths::datasets("glass"), "Type");
+    arff.load(Paths::datasets("glass"), std::string("Type"));
     REQUIRE(arff.getClassName() == "Type");
     REQUIRE(arff.getClassType() == "{ 'build wind float', 'build wind non-float', 'vehic wind float', 'vehic wind non-float', containers, tableware, headlamps}");
     REQUIRE(arff.getLabels().size() == 6);
-    REQUIRE(arff.getLabels()[0] == "'build wind float'");
+    REQUIRE(arff.getLabels()[0] == "build wind float");
-    REQUIRE(arff.getLabels()[1] == "'vehic wind float'");
+    REQUIRE(arff.getLabels()[1] == "vehic wind float");
     REQUIRE(arff.getLabels()[2] == "tableware");
-    REQUIRE(arff.getLabels()[3] == "'build wind non-float'");
+    REQUIRE(arff.getLabels()[3] == "build wind non-float");
     REQUIRE(arff.getLabels()[4] == "headlamps");
     REQUIRE(arff.getLabels()[5] == "containers");
     REQUIRE(arff.getSize() == 214);
@@ -119,14 +119,34 @@ TEST_CASE("Load with class name as first attribute", "[ArffFiles]")
 TEST_CASE("Adult dataset", "[ArffFiles]")
 {
     ArffFiles arff;
-    arff.load(Paths::datasets("adult"));
+    arff.load(Paths::datasets("adult"), std::string("class"));
     REQUIRE(arff.getClassName() == "class");
-    REQUIRE(arff.getClassType() == "{ <=50K, >50K}");
+    REQUIRE(arff.getClassType() == "{ >50K, <=50K }");
     REQUIRE(arff.getLabels().size() == 2);
     REQUIRE(arff.getLabels()[0] == "<=50K");
     REQUIRE(arff.getLabels()[1] == ">50K");
-    REQUIRE(arff.getSize() == 32561);
+    REQUIRE(arff.getSize() == 45222);
-    REQUIRE(arff.getLines().size() == 32561);
+    REQUIRE(arff.getLines().size() == 45222);
-    REQUIRE(arff.getLines()[0] == "39, State-gov, 77516, Bachelors, 13, Never-married, Adm-clerical, Not-in-family, White   ");
+    REQUIRE(arff.getLines()[0] == "25, Private, 226802, 11th, 7, Never-married, Machine-op-inspct, Own-child, Black, Male, 0, 0, 40, United-States, <=50K");
+    auto X = arff.getX();
+    REQUIRE(X[0][0] == 25);
+    REQUIRE(X[1][0] == 0);
+    REQUIRE(X[2][0] == 226802);
+    auto states = arff.getStates();
+    auto numeric = arff.getNumericAttributes();
+    auto attributes = arff.getAttributes();
+    for (size_t i = 0; i < numeric.size(); ++i) {
+        auto feature = attributes.at(i).first;
+        auto state = states.at(feature);
+        if (!numeric.at(i)) {
+            std::cout << feature << ": ";
+            for (const auto& s : state) {
+                std::cout << s << ", ";
+            }
+            std::cout << std::endl;
+        } else {
+            std::cout << feature << " size: " << state.size() << std::endl;
+        }
+    }
 }