Doctorado-ML/benchmark
1
0
Fork 0
mirror of https://github.com/Doctorado-ML/benchmark.git synced 2025-08-16 16:05:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix stochastic error in discretization

Browse Source
This commit is contained in:
Ricardo Montañana Gómez
2022-11-14 21:51:53 +01:00
parent a2db4f1f6d
commit a53e957c00
2 changed files with 26 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
benchmark/Datasets.py
View File

@@ -26,7 +26,7 @@ class DatasetsArff:
def folder(): def folder():
return "datasets" return "datasets"
def load(self, name, class_name, dataframe): def load(self, name, class_name):
file_name = os.path.join(self.folder(), self.dataset_names(name)) file_name = os.path.join(self.folder(), self.dataset_names(name))
data = arff.loadarff(file_name) data = arff.loadarff(file_name)
df = pd.DataFrame(data[0]) df = pd.DataFrame(data[0])
@@ -35,9 +35,8 @@ class DatasetsArff:
self.features = X.columns self.features = X.columns
self.class_name = class_name self.class_name = class_name
y, _ = pd.factorize(df[class_name]) y, _ = pd.factorize(df[class_name])
df[class_name] = y
X = X.to_numpy() X = X.to_numpy()
return df if dataframe else (X, y) return X, y
class DatasetsTanveer: class DatasetsTanveer:
@@ -149,10 +148,10 @@ class Datasets:
def get_class_name(self): def get_class_name(self):
return self.dataset.class_name return self.dataset.class_name
def load_continuous(self, name, dataframe=False): def load_continuous(self, name):
try: try:
class_name = self.class_names[self.data_sets.index(name)] class_name = self.class_names[self.data_sets.index(name)]
return self.dataset.load(name, class_name, dataframe) return self.dataset.load(name, class_name)
except (ValueError, FileNotFoundError): except (ValueError, FileNotFoundError):
raise ValueError(f"Unknown dataset: {name}") raise ValueError(f"Unknown dataset: {name}")
@@ -170,16 +169,18 @@ class Datasets:
------- -------
tuple (X, y) of numpy.ndarray tuple (X, y) of numpy.ndarray
""" """
discretiz = MDLP() discretiz = MDLP(random_state=17, dtype=np.int32)
Xdisc = discretiz.fit_transform(X, y) Xdisc = discretiz.fit_transform(X, y)
return Xdisc.astype(int), y.astype(int) return Xdisc
def load_discretized(self, name, dataframe=False): def load_discretized(self, name, dataframe=False):
X, y = self.load_continuous(name) X, yd = self.load_continuous(name)
X, y = self.discretize(X, y) Xd = self.discretize(X, yd)
dataset = pd.DataFrame(X, columns=self.get_features()) dataset = pd.DataFrame(Xd, columns=self.get_features())
dataset[self.get_class_name()] = y dataset[self.get_class_name()] = yd
return dataset if dataframe else X, y if dataframe:
return dataset
return Xd, yd
def __iter__(self) -> Diterator: def __iter__(self) -> Diterator:
return Diterator(self.data_sets) return Diterator(self.data_sets)

13
benchmark/tests/Dataset_test.py
View File

@@ -30,6 +30,19 @@ class DatasetTest(TestBase):
expected = [271, 314, 171] expected = [271, 314, 171]
self.assertSequenceEqual(Randomized.seeds(), expected) self.assertSequenceEqual(Randomized.seeds(), expected)
def test_load_dataframe(self):
self.set_env(".env.arff")
dt = Datasets()
X, y = dt.load_discretized("iris", dataframe=False)
dataset = dt.load_discretized("iris", dataframe=True)
class_name = dt.get_class_name()
features = dt.get_features()
self.assertListEqual(y.tolist(), dataset[class_name].tolist())
for i in range(len(features)):
self.assertListEqual(
X[:, i].tolist(), dataset[features[i]].tolist()
)
def test_Datasets_iterator(self): def test_Datasets_iterator(self):
test = { test = {
".env.dist": ["balance-scale", "balloons"], ".env.dist": ["balance-scale", "balloons"],