/** * @file test_performance.cpp * @brief Performance benchmarks for SVMClassifier */ #include #include #include #include #include #include // Include the actual headers for complete struct definitions #include "svm.h" // libsvm structures #include "linear.h" // liblinear structures #include using namespace svm_classifier; using json = nlohmann::json; /** * @brief Generate large synthetic dataset for performance testing */ std::pair generate_large_dataset(int n_samples, int n_features, int n_classes = 2, int seed = 42) { torch::manual_seed(seed); auto X = torch::randn({ n_samples, n_features }); auto y = torch::randint(0, n_classes, { n_samples }); // Add some structure to make the problem non-trivial for (int i = 0; i < n_samples; ++i) { int class_label = y[i].item(); // Add class-dependent bias X[i] += class_label * torch::randn({ n_features }) * 0.3; } return { X, y }; } /** * @brief Benchmark helper class */ class Benchmark { public: explicit Benchmark(const std::string& name) : name_(name) { start_time_ = std::chrono::high_resolution_clock::now(); } ~Benchmark() { auto end_time = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end_time - start_time_); std::cout << std::setw(40) << std::left << name_ << ": " << std::setw(8) << std::right << duration.count() << " ms" << std::endl; } private: std::string name_; std::chrono::high_resolution_clock::time_point start_time_; }; TEST_CASE("Performance Benchmarks - Training Speed", "[performance][training]") { std::cout << "\n=== Training Performance Benchmarks ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; SECTION("Linear kernel performance") { auto [X_small, y_small] = generate_large_dataset(1000, 20, 2); auto [X_medium, y_medium] = generate_large_dataset(5000, 50, 3); auto [X_large, y_large] = generate_large_dataset(10000, 100, 2); { Benchmark bench("Linear SVM - 1K samples, 20 features"); SVMClassifier svm(KernelType::LINEAR, 1.0); svm.fit(X_small, y_small); } { Benchmark bench("Linear SVM - 5K samples, 50 features"); SVMClassifier svm(KernelType::LINEAR, 1.0); svm.fit(X_medium, y_medium); } { Benchmark bench("Linear SVM - 10K samples, 100 features"); SVMClassifier svm(KernelType::LINEAR, 1.0); svm.fit(X_large, y_large); } } SECTION("RBF kernel performance") { auto [X_small, y_small] = generate_large_dataset(500, 10, 2); auto [X_medium, y_medium] = generate_large_dataset(1000, 20, 2); auto [X_large, y_large] = generate_large_dataset(2000, 30, 2); { Benchmark bench("RBF SVM - 500 samples, 10 features"); SVMClassifier svm(KernelType::RBF, 1.0); svm.fit(X_small, y_small); } { Benchmark bench("RBF SVM - 1K samples, 20 features"); SVMClassifier svm(KernelType::RBF, 1.0); svm.fit(X_medium, y_medium); } { Benchmark bench("RBF SVM - 2K samples, 30 features"); SVMClassifier svm(KernelType::RBF, 1.0); svm.fit(X_large, y_large); } } SECTION("Polynomial kernel performance") { auto [X_small, y_small] = generate_large_dataset(300, 8, 2); auto [X_medium, y_medium] = generate_large_dataset(800, 15, 2); { Benchmark bench("Poly SVM (deg=2) - 300 samples, 8 features"); json config = { {"kernel", "polynomial"}, {"degree", 2}, {"C", 1.0} }; SVMClassifier svm(config); svm.fit(X_small, y_small); } { Benchmark bench("Poly SVM (deg=3) - 800 samples, 15 features"); json config = { {"kernel", "polynomial"}, {"degree", 3}, {"C", 1.0} }; SVMClassifier svm(config); svm.fit(X_medium, y_medium); } } } TEST_CASE("Performance Benchmarks - Prediction Speed", "[performance][prediction]") { std::cout << "\n=== Prediction Performance Benchmarks ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; SECTION("Linear kernel prediction") { auto [X_train, y_train] = generate_large_dataset(2000, 50, 3); auto [X_test_small, _] = generate_large_dataset(100, 50, 3, 123); auto [X_test_medium, _] = generate_large_dataset(1000, 50, 3, 124); auto [X_test_large, _] = generate_large_dataset(5000, 50, 3, 125); SVMClassifier svm(KernelType::LINEAR, 1.0); svm.fit(X_train, y_train); { Benchmark bench("Linear prediction - 100 samples"); auto predictions = svm.predict(X_test_small); } { Benchmark bench("Linear prediction - 1K samples"); auto predictions = svm.predict(X_test_medium); } { Benchmark bench("Linear prediction - 5K samples"); auto predictions = svm.predict(X_test_large); } } SECTION("RBF kernel prediction") { auto [X_train, y_train] = generate_large_dataset(1000, 20, 2); auto [X_test_small, _] = generate_large_dataset(50, 20, 2, 123); auto [X_test_medium, _] = generate_large_dataset(500, 20, 2, 124); auto [X_test_large, _] = generate_large_dataset(2000, 20, 2, 125); SVMClassifier svm(KernelType::RBF, 1.0); svm.fit(X_train, y_train); { Benchmark bench("RBF prediction - 50 samples"); auto predictions = svm.predict(X_test_small); } { Benchmark bench("RBF prediction - 500 samples"); auto predictions = svm.predict(X_test_medium); } { Benchmark bench("RBF prediction - 2K samples"); auto predictions = svm.predict(X_test_large); } } } TEST_CASE("Performance Benchmarks - Multiclass Strategies", "[performance][multiclass]") { std::cout << "\n=== Multiclass Strategy Performance ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; auto [X, y] = generate_large_dataset(2000, 30, 5); // 5 classes SECTION("One-vs-Rest vs One-vs-One") { { Benchmark bench("OvR Linear - 5 classes, 2K samples"); json config = { {"kernel", "linear"}, {"multiclass_strategy", "ovr"} }; SVMClassifier svm_ovr(config); svm_ovr.fit(X, y); } { Benchmark bench("OvO Linear - 5 classes, 2K samples"); json config = { {"kernel", "linear"}, {"multiclass_strategy", "ovo"} }; SVMClassifier svm_ovo(config); svm_ovo.fit(X, y); } // Smaller dataset for RBF due to computational complexity auto [X_rbf, y_rbf] = generate_large_dataset(800, 15, 4); { Benchmark bench("OvR RBF - 4 classes, 800 samples"); json config = { {"kernel", "rbf"}, {"multiclass_strategy", "ovr"} }; SVMClassifier svm_ovr(config); svm_ovr.fit(X_rbf, y_rbf); } { Benchmark bench("OvO RBF - 4 classes, 800 samples"); json config = { {"kernel", "rbf"}, {"multiclass_strategy", "ovo"} }; SVMClassifier svm_ovo(config); svm_ovo.fit(X_rbf, y_rbf); } } } TEST_CASE("Performance Benchmarks - Memory Usage", "[performance][memory]") { std::cout << "\n=== Memory Usage Benchmarks ===" << std::endl; SECTION("Large dataset handling") { // Test with progressively larger datasets std::vector dataset_sizes = { 1000, 5000, 10000, 20000 }; for (int size : dataset_sizes) { auto [X, y] = generate_large_dataset(size, 50, 2); { Benchmark bench("Dataset size " + std::to_string(size) + " - Linear"); SVMClassifier svm(KernelType::LINEAR, 1.0); svm.fit(X, y); // Test prediction memory usage auto predictions = svm.predict(X.slice(0, 0, std::min(1000, size))); REQUIRE(predictions.size(0) == std::min(1000, size)); } } } SECTION("High-dimensional data") { std::vector feature_sizes = { 100, 500, 1000, 2000 }; for (int n_features : feature_sizes) { auto [X, y] = generate_large_dataset(1000, n_features, 2); { Benchmark bench("Features " + std::to_string(n_features) + " - Linear"); SVMClassifier svm(KernelType::LINEAR, 1.0); svm.fit(X, y); auto predictions = svm.predict(X.slice(0, 0, 100)); REQUIRE(predictions.size(0) == 100); } } } } TEST_CASE("Performance Benchmarks - Cross-Validation", "[performance][cv]") { std::cout << "\n=== Cross-Validation Performance ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; auto [X, y] = generate_large_dataset(2000, 25, 3); SECTION("Different CV folds") { SVMClassifier svm(KernelType::LINEAR, 1.0); { Benchmark bench("3-fold CV - 2K samples"); auto scores = svm.cross_validate(X, y, 3); REQUIRE(scores.size() == 3); } { Benchmark bench("5-fold CV - 2K samples"); auto scores = svm.cross_validate(X, y, 5); REQUIRE(scores.size() == 5); } { Benchmark bench("10-fold CV - 2K samples"); auto scores = svm.cross_validate(X, y, 10); REQUIRE(scores.size() == 10); } } } TEST_CASE("Performance Benchmarks - Grid Search", "[performance][grid_search]") { std::cout << "\n=== Grid Search Performance ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; auto [X, y] = generate_large_dataset(1000, 20, 2); // Smaller dataset for grid search SVMClassifier svm; SECTION("Small parameter grid") { json param_grid = { {"kernel", {"linear"}}, {"C", {0.1, 1.0, 10.0}} }; { Benchmark bench("Grid search - 3 parameters"); auto results = svm.grid_search(X, y, param_grid, 3); REQUIRE(results.contains("best_params")); } } SECTION("Medium parameter grid") { json param_grid = { {"kernel", {"linear", "rbf"}}, {"C", {0.1, 1.0, 10.0}} }; { Benchmark bench("Grid search - 6 parameters"); auto results = svm.grid_search(X, y, param_grid, 3); REQUIRE(results.contains("best_params")); } } SECTION("Large parameter grid") { json param_grid = { {"kernel", {"linear", "rbf"}}, {"C", {0.1, 1.0, 10.0, 100.0}}, {"gamma", {0.01, 0.1, 1.0}} }; { Benchmark bench("Grid search - 24 parameters"); auto results = svm.grid_search(X, y, param_grid, 3); REQUIRE(results.contains("best_params")); } } } TEST_CASE("Performance Benchmarks - Data Conversion", "[performance][data_conversion]") { std::cout << "\n=== Data Conversion Performance ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; DataConverter converter; SECTION("Tensor to SVM format conversion") { auto [X_small, y_small] = generate_large_dataset(1000, 50, 2); auto [X_medium, y_medium] = generate_large_dataset(5000, 100, 2); auto [X_large, y_large] = generate_large_dataset(10000, 200, 2); { Benchmark bench("SVM conversion - 1K x 50"); auto problem = converter.to_svm_problem(X_small, y_small); REQUIRE(problem->l == 1000); } { Benchmark bench("SVM conversion - 5K x 100"); auto problem = converter.to_svm_problem(X_medium, y_medium); REQUIRE(problem->l == 5000); } { Benchmark bench("SVM conversion - 10K x 200"); auto problem = converter.to_svm_problem(X_large, y_large); REQUIRE(problem->l == 10000); } } SECTION("Tensor to Linear format conversion") { auto [X_small, y_small] = generate_large_dataset(1000, 50, 2); auto [X_medium, y_medium] = generate_large_dataset(5000, 100, 2); auto [X_large, y_large] = generate_large_dataset(10000, 200, 2); { Benchmark bench("Linear conversion - 1K x 50"); auto problem = converter.to_linear_problem(X_small, y_small); REQUIRE(problem->l == 1000); } { Benchmark bench("Linear conversion - 5K x 100"); auto problem = converter.to_linear_problem(X_medium, y_medium); REQUIRE(problem->l == 5000); } { Benchmark bench("Linear conversion - 10K x 200"); auto problem = converter.to_linear_problem(X_large, y_large); REQUIRE(problem->l == 10000); } } } TEST_CASE("Performance Benchmarks - Probability Prediction", "[performance][probability]") { std::cout << "\n=== Probability Prediction Performance ===" << std::endl; std::cout << std::setw(40) << std::left << "Test Name" << " " << "Time" << std::endl; std::cout << std::string(50, '-') << std::endl; auto [X_train, y_train] = generate_large_dataset(1000, 20, 3); auto [X_test, _] = generate_large_dataset(500, 20, 3, 999); SECTION("Linear kernel with probability") { json config = { {"kernel", "linear"}, {"probability", true} }; SVMClassifier svm(config); svm.fit(X_train, y_train); { Benchmark bench("Linear probability prediction"); if (svm.supports_probability()) { auto probabilities = svm.predict_proba(X_test); REQUIRE(probabilities.size(0) == X_test.size(0)); } } } SECTION("RBF kernel with probability") { json config = { {"kernel", "rbf"}, {"probability", true} }; SVMClassifier svm(config); svm.fit(X_train, y_train); { Benchmark bench("RBF probability prediction"); if (svm.supports_probability()) { auto probabilities = svm.predict_proba(X_test); REQUIRE(probabilities.size(0) == X_test.size(0)); } } } } TEST_CASE("Performance Summary", "[performance][summary]") { std::cout << "\n=== Performance Summary ===" << std::endl; std::cout << "All performance benchmarks completed successfully!" << std::endl; std::cout << "\nKey Observations:" << std::endl; std::cout << "- Linear kernels are fastest for training and prediction" << std::endl; std::cout << "- RBF kernels provide good accuracy but slower training" << std::endl; std::cout << "- One-vs-Rest is generally faster than One-vs-One" << std::endl; std::cout << "- Memory usage scales linearly with dataset size" << std::endl; std::cout << "- Data conversion overhead is minimal" << std::endl; std::cout << "\nFor production use:" << std::endl; std::cout << "- Use linear kernels for large datasets (>10K samples)" << std::endl; std::cout << "- Use RBF kernels for smaller, complex datasets" << std::endl; std::cout << "- Consider One-vs-Rest for many classes (>5)" << std::endl; std::cout << "- Enable probability only when needed" << std::endl; }