diff --git a/include/analysis/Grader.h b/include/analysis/Grader.h index 3a166303..75b47345 100644 --- a/include/analysis/Grader.h +++ b/include/analysis/Grader.h @@ -24,7 +24,6 @@ class Grader : public TestHarness { Grader(const Config& cfg) : TestHarness(cfg), failedTestLog(*cfg.getFailureLogPath()), solutionExecutable(*cfg.getSolutionExecutable()) { - findTests(); buildResults(); } diff --git a/include/config/Config.h b/include/config/Config.h index 2979d767..6097e983 100644 --- a/include/config/Config.h +++ b/include/config/Config.h @@ -3,6 +3,7 @@ #include "toolchain/ToolChain.h" +#include #include #include #include @@ -53,10 +54,13 @@ class Config { // Config int getters. int64_t getTimeout() const { return timeout; } + int64_t getNumThreads() const { return numThreads; } + int8_t getBatchSize() const { return batchSize; } // Initialisation verification. bool isInitialised() const { return initialised; } int getErrorCode() const { return errorCode; } + private: // Option file paths. @@ -82,6 +86,11 @@ class Config { // The command timeout. int64_t timeout; + // Number of threads on which to run tests + int64_t numThreads; + // Number of tests for each thread to grab on each run + int8_t batchSize; + // Is the config initialised or not and an appropriate error code. This // could be due to asking for help or a missing config file. bool initialised; diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h index ac32e329..3c849fa8 100644 --- a/include/testharness/TestHarness.h +++ b/include/testharness/TestHarness.h @@ -5,6 +5,7 @@ #include "config/Config.h" #include "testharness/ResultManager.h" #include "tests/TestParser.h" +#include "tests/TestResult.h" #include "toolchain/ToolChain.h" #include @@ -17,7 +18,8 @@ namespace fs = std::filesystem; namespace tester { // Test hierarchy types -typedef std::vector> SubPackage; +typedef std::pair, std::optional> TestPair; +typedef std::vector SubPackage; typedef std::map Package; typedef std::map TestSet; @@ -49,19 +51,33 @@ class TestHarness { // A separate subpackage, just for invalid tests. SubPackage invalidTests; +protected: // let derived classes find tests. void findTests(); private: // The results of the tests. + // NOTE we keep both a result manager and + // the result in the TestSet to ensure in-ordre + // printing ResultManager results; private: + // thread control + void spawnThreads(); + // test running - bool runTestsForToolChain(std::string tcId, std::string exeName); + void threadRunTestBatch(std::reference_wrapper> toolchains, + std::reference_wrapper> executables, + std::reference_wrapper>> tests, + std::reference_wrapper currentIndex, std::reference_wrapper lock); + void threadRunTestsForToolChain(std::reference_wrapper> tcIds, + std::reference_wrapper> exeNames, + std::reference_wrapper>> tests, size_t begin, size_t end); // helper for formatting tester output void printTestResult(const TestFile *test, TestResult result); + bool aggregateTestResultsForToolChain(std::string tcName, std::string exeName); // test finding and filling methods void addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file); diff --git a/include/tests/TestResult.h b/include/tests/TestResult.h index 62ff9c19..8a1621a1 100644 --- a/include/tests/TestResult.h +++ b/include/tests/TestResult.h @@ -17,12 +17,21 @@ struct TestResult { : name(in.stem()), pass(pass), error(error), diff(diff) {} // Info about result. - const fs::path name; - const bool pass; - const bool error; - const std::string diff; -}; + fs::path name; + bool pass; + bool error; + std::string diff; + + TestResult clone() { return TestResult(this->name, this->pass, this->error, this->diff); } + void swap(TestResult &other) { + std::swap(name, other.name); + std::swap(pass, other.pass); + std::swap(error, other.error); + std::swap(diff, other.diff); + } + +}; } // End namespace tester #endif // TESTER_TEST_RESULT_H diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp index b9ddcefd..e32d3911 100644 --- a/src/analysis/Grader.cpp +++ b/src/analysis/Grader.cpp @@ -1,6 +1,7 @@ #include "analysis/Grader.h" #include #include +#include namespace { @@ -125,10 +126,16 @@ void Grader::fillToolchainResultsJSON() { // attacker, tracking pass count. size_t passCount = 0, testCount = 0; for (const auto& subpackages : testSet[attacker]) { - for (const std::unique_ptr& test : subpackages.second) { + for (const TestPair& testpair : subpackages.second) { + const std::unique_ptr& test = testpair.first; + // Poll while we wait for the result + // TODO this could probably be replaced with some sort of interrupt, + // (and probably should be), but better this than no threads + while (!testpair.second.has_value()) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + TestResult result = testpair.second.value(); - TestResult result = runTest(test.get(), tc, cfg); - if (!result.pass && defender == solutionExecutable) { if ( attacker == solutionExecutable ) { // A testcase just failed the solution executable @@ -178,4 +185,4 @@ void Grader::buildResults() { fillToolchainResultsJSON(); } -} // End namespace tester \ No newline at end of file +} // End namespace tester diff --git a/src/config/Config.cpp b/src/config/Config.cpp index 15221ad0..c5051e45 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -1,6 +1,7 @@ #include "config/Config.h" #include "util.h" +#include "Colors.h" #include "CLI11.hpp" @@ -8,12 +9,15 @@ #include +#define WARN(msg) \ + std::cout << Colors::YELLOW << "WARNING: " << Colors::RESET << msg << std::endl; + // Convenience. using JSON = nlohmann::json; namespace tester { -Config::Config(int argc, char** argv) : timeout(2l) { +Config::Config(int argc, char** argv) : timeout(2l), numThreads(1), batchSize(5) { CLI::App app{"CMPUT 415 testing utility"}; @@ -33,7 +37,11 @@ Config::Config(int argc, char** argv) : timeout(2l) { app.add_flag("-t,--time", time, "Include the timings (seconds) of each test in the output."); app.add_flag_function("-v", [&](size_t count) { verbosity = static_cast(count); }, "Increase verbosity level"); - + + // multithreading options + app.add_option("-j", numThreads, "The number of threads on which to execute tests."); + app.add_option("--batch-size", batchSize, "(ADVANCED) the number of tests for each thread to grab per iteration. (default = 5)"); + // Enforce that if a grade path is supplied, then a log file should be as well and vice versa gradeOpt->needs(solutionFailureLogOpt); solutionFailureLogOpt->needs(gradeOpt); @@ -106,6 +114,12 @@ Config::Config(int argc, char** argv) : timeout(2l) { for (auto it = tcJson.begin(); it != tcJson.end(); ++it) { toolchains.emplace(std::make_pair(it.key(), ToolChain(it.value(), timeout))); } + + if (numThreads < 1) + throw std::runtime_error("Cannot execute on less than one thread."); + + if (numThreads > std::thread::hardware_concurrency()) + WARN("More threads than hardware supported concurrent threads, performance may suffer"); } -} // namespace tester \ No newline at end of file +} // namespace tester diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp index a6bc59c5..1ec8c217 100644 --- a/src/testharness/TestHarness.cpp +++ b/src/testharness/TestHarness.cpp @@ -4,27 +4,137 @@ #include "tests/TestRunning.h" #include "util.h" +#include #include #include +#include +#include #include +#include #include namespace tester { +void swap(TestResult& first, TestResult& second) { + std::swap(first, second); +} + // Builds TestSet during object creation. bool TestHarness::runTests() { + // initialize the threads + std::thread t(&TestHarness::spawnThreads, this); + bool failed = false; // Iterate over executables. for (auto exePair : cfg.getExecutables()) { // Iterate over toolchains. for (auto& tcPair : cfg.getToolChains()) { - if (runTestsForToolChain(exePair.first, tcPair.first) == 1) + if (aggregateTestResultsForToolChain(tcPair.first, exePair.first) == 1) failed = true; } } + + // join the control thread + t.join(); return failed; } +void TestHarness::spawnThreads() { + std::vector threadPool; + std::vector> flattenedList; + std::vector exeList; + std::vector tcList; + + static size_t currentIndex = 0; + std::mutex currentIndexLock; + + // Initialize the threads + // Iterate over executables. + for (auto exePair : cfg.getExecutables()) { + // Iterate over toolchains. + for (auto& tcPair : cfg.getToolChains()) { + // iterate over packages + for (auto& package : testSet) { // TestSet.second -> Package + // iterate over subpackages + for (auto& subpackage : package.second) { // Package.second -> SubPackage + // populate the flattened test vector + for (auto& test : subpackage.second) { + flattenedList.push_back(std::ref(test)); + exeList.push_back(exePair.first); + tcList.push_back(tcPair.first); + } + } + } + } + } + + // Let the load balancing be the responsability of the threads, instead of the supervisor + for (int64_t i = 0; i < cfg.getNumThreads(); i++) { + std::thread t(&TestHarness::threadRunTestBatch, this, + std::ref(tcList), + std::ref(exeList), + std::ref(flattenedList), + std::ref(currentIndex), + std::ref(currentIndexLock)); + threadPool.push_back(std::move(t)); + } + + // KILL ALL THE CHILDREN + for (auto& thread : threadPool) thread.join(); +} + +void TestHarness::threadRunTestBatch(std::reference_wrapper> toolchains, + std::reference_wrapper> executables, + std::reference_wrapper>> tests, + std::reference_wrapper currentIndex, + std::reference_wrapper currentIndexLock) +{ + // Loop while we have not exhausted the available tests + while (currentIndex < tests.get().size()) { + size_t tmpIndex; + + { + // Lock the index + std::lock_guard lock(currentIndexLock); + + // store the current index + tmpIndex = currentIndex; + // increment for the next lad + currentIndex += cfg.getBatchSize(); + } + size_t endIndex = ((tmpIndex + cfg.getBatchSize()) >= tests.get().size()) ? tests.get().size() : tmpIndex + cfg.getBatchSize(); + + threadRunTestsForToolChain(std::ref(toolchains), std::ref(executables), std::ref(tests), tmpIndex, endIndex); + } +} + +void TestHarness::threadRunTestsForToolChain(std::reference_wrapper> tcNames, + std::reference_wrapper> exeNames, + std::reference_wrapper>> tests, + size_t begin, size_t end) +{ + for (size_t i = begin; i < end; i++) { + ToolChain toolChain = cfg.getToolChain(tcNames.get().at(i)); // Get the toolchain to use. + const fs::path& exe = cfg.getExecutablePath(exeNames.get().at(i)); // Set the toolchain's exe to be tested. + toolChain.setTestedExecutable(exe); + + // set the runtime + if (cfg.hasRuntime(exeNames.get().at(i))) // If we have a runtime, set that as well. + toolChain.setTestedRuntime(cfg.getRuntimePath(exeNames.get().at(i))); + else + toolChain.setTestedRuntime(""); + + std::unique_ptr& test = tests.get().at(i).get().first; + if (test->getParseError() == ParseError::NoError) { + + TestResult result = runTest(test.get(), toolChain, cfg); + // keep the result with the test for pretty printing + std::optional res_clone = std::make_optional(result.clone()); + tests.get().at(i).get().second.swap(res_clone); + } + } +} + std::string TestHarness::getTestInfo() const { std::ostringstream oss; oss << "Test Information:\n\n"; @@ -54,18 +164,13 @@ void TestHarness::printTestResult(const TestFile *test, TestResult result) { std::cout << "\n"; } -bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) { +bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::string exeName) { bool failed = false; ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use. const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested. toolChain.setTestedExecutable(exe); - if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well. - toolChain.setTestedRuntime(cfg.getRuntimePath(exeName)); - else - toolChain.setTestedRuntime(""); - std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n'; std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n'; @@ -83,12 +188,24 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) // Iterate over each test in the package for (size_t i = 0; i < subPackage.size(); ++i) { - std::unique_ptr& test = subPackage[i]; + TestPair& pair = subPackage[i]; + std::unique_ptr& test = pair.first; if (test->getParseError() == ParseError::NoError) { - - TestResult result = runTest(test.get(), toolChain, cfg); + + // Poll while we wait for the result + // TODO this could probably be replaced with some sort of interrupt, + // (and probably should be), but better this than no threads + while (!pair.second.has_value()) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + TestResult result = pair.second.value(); + + // keep the result with the test for pretty printing + std::optional res_clone = std::make_optional(result.clone()); + subPackage[i].second.swap(res_clone); + results.addResult(exeName, tcName, subPackageName, result); - printTestResult(test.get(), result); + printTestResult(test.get(), result); if (result.pass) { ++packagePasses; @@ -100,7 +217,7 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) std::cout << " " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) << " " << test->getTestPath().stem().string() << '\n'; --subPackageSize; - } + } } std::cout << " Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n'; // Track how many tests we run. @@ -119,11 +236,14 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) << "\n"; for (auto& test : invalidTests) { - std::cout << " Skipped: " << test->getTestPath().filename().stem() << std::endl - << " Error: " << Colors::YELLOW << test->getParseErrorMsg() << Colors::RESET << "\n"; + std::cout << " Skipped: " << test.first->getTestPath().filename().stem() << std::endl + << " Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n"; } std::cout << "\n"; + std::cout << Colors::GREEN << "Completed Tests" << Colors::RESET << std::endl; + std::cout << "Hold on while we clean up any remaining threads, this might take a moment" << std::endl; + return failed; } @@ -150,10 +270,11 @@ void TestHarness::addTestFileToSubPackage(SubPackage& subPackage, const fs::path TestParser parser(testfile.get()); + std::optional no_result = std::nullopt; if (testfile->didError()) { - invalidTests.push_back(std::move(testfile)); - } else { - subPackage.push_back(std::move(testfile)); + invalidTests.push_back({std::move(testfile), no_result}); + }else { + subPackage.push_back({std::move(testfile), no_result}); } } diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp index a4ac1015..c010f412 100644 --- a/src/tests/TestRunning.cpp +++ b/src/tests/TestRunning.cpp @@ -245,4 +245,4 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg return TestResult(testPath, !testDiff, testError, ""); } -} // End namespace tester \ No newline at end of file +} // End namespace tester