Multithreading Implementation

include/analysis/Grader.h

@@ -24,7 +24,6 @@ class Grader : public TestHarness {
   Grader(const Config& cfg) : TestHarness(cfg), 
                               failedTestLog(*cfg.getFailureLogPath()),
                               solutionExecutable(*cfg.getSolutionExecutable()) {
-    findTests();
     buildResults();
   }
 

include/config/Config.h

@@ -3,6 +3,7 @@
 
 #include "toolchain/ToolChain.h"
 
+#include <cstdint>
 #include <filesystem>
 #include <map>
 #include <string>
@@ -53,10 +54,13 @@ class Config {
 
   // Config int getters.
   int64_t getTimeout() const { return timeout; }
+  int64_t getNumThreads() const { return numThreads; }
+  int8_t getBatchSize() const { return batchSize; }
 
   // Initialisation verification.
   bool isInitialised() const { return initialised; }
   int getErrorCode() const { return errorCode; }
+
 
 private:
   // Option file paths.
@@ -82,6 +86,11 @@ class Config {
   // The command timeout.
   int64_t timeout;
 
+  // Number of threads on which to run tests
+  int64_t numThreads;
+  // Number of tests for each thread to grab on each run
+  int8_t batchSize;
+
   // Is the config initialised or not and an appropriate error code. This
   // could be due to asking for help or a missing config file.
   bool initialised;

include/testharness/TestHarness.h

@@ -5,6 +5,7 @@
 #include "config/Config.h"
 #include "testharness/ResultManager.h"
 #include "tests/TestParser.h"
+#include "tests/TestResult.h"
 #include "toolchain/ToolChain.h"
 
 #include <filesystem>
@@ -17,7 +18,8 @@ namespace fs = std::filesystem;
 namespace tester {
 
 // Test hierarchy types
-typedef std::vector<std::unique_ptr<TestFile>> SubPackage;
+typedef std::pair<std::unique_ptr<TestFile>, std::optional<TestResult>> TestPair;
+typedef std::vector<TestPair> SubPackage;
 typedef std::map<std::string, SubPackage> Package;
 typedef std::map<std::string, Package> TestSet;
 
@@ -49,19 +51,33 @@ class TestHarness {
   // A separate subpackage, just for invalid tests.
   SubPackage invalidTests;
 
+protected:
   // let derived classes find tests.
   void findTests();
 
 private:
   // The results of the tests.
+  //  NOTE we keep both a result manager and
+  //  the result in the TestSet to ensure in-ordre
+  //  printing
   ResultManager results;
 
 private:
+  // thread control
+  void spawnThreads();
+
   // test running
-  bool runTestsForToolChain(std::string tcId, std::string exeName);
+  void threadRunTestBatch(std::reference_wrapper<std::vector<std::string>> toolchains,
+                          std::reference_wrapper<std::vector<std::string>> executables,
+                          std::reference_wrapper<std::vector<std::reference_wrapper<TestPair>>> tests,
+                          std::reference_wrapper<size_t> currentIndex, std::reference_wrapper<std::mutex> lock);
+  void threadRunTestsForToolChain(std::reference_wrapper<std::vector<std::string>> tcIds,
+                                  std::reference_wrapper<std::vector<std::string>> exeNames,
+                                  std::reference_wrapper<std::vector<std::reference_wrapper<TestPair>>> tests, size_t begin, size_t end);
 
   // helper for formatting tester output 
   void printTestResult(const TestFile *test, TestResult result);
+  bool aggregateTestResultsForToolChain(std::string tcName, std::string exeName);
 
   // test finding and filling methods
   void addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file);

include/tests/TestResult.h

@@ -17,12 +17,21 @@ struct TestResult {
       : name(in.stem()), pass(pass), error(error), diff(diff) {}
 
   // Info about result.
-  const fs::path name;
-  const bool pass;
-  const bool error;
-  const std::string diff;
-};
+  fs::path name;
+  bool pass;
+  bool error;
+  std::string diff;
+
+  TestResult clone() { return TestResult(this->name, this->pass, this->error, this->diff); }
 
+  void swap(TestResult &other) {
+    std::swap(name, other.name);
+    std::swap(pass, other.pass);
+    std::swap(error, other.error);
+    std::swap(diff, other.diff);
+  }
+
+};
 } // End namespace tester
 
 #endif // TESTER_TEST_RESULT_H

src/analysis/Grader.cpp

@@ -1,6 +1,7 @@
 #include "analysis/Grader.h"
 #include <algorithm>
 #include <iostream>
+#include <thread>
 
 namespace {
 
@@ -125,10 +126,16 @@ void Grader::fillToolchainResultsJSON() {
         // attacker, tracking pass count.
         size_t passCount = 0, testCount = 0;
         for (const auto& subpackages : testSet[attacker]) {
-          for (const std::unique_ptr<TestFile>& test : subpackages.second) {
+          for (const TestPair& testpair : subpackages.second) {
+            const std::unique_ptr<TestFile>& test = testpair.first;
+            // Poll while we wait for the result
+            // TODO this could probably be replaced with some sort of interrupt,
+            //  (and probably should be), but better this than no threads
+            while (!testpair.second.has_value())
+              std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+            TestResult result = testpair.second.value();
 
-            TestResult result = runTest(test.get(), tc, cfg);
-
             if (!result.pass && defender == solutionExecutable) {
               if ( attacker == solutionExecutable ) {
                 // A testcase just failed the solution executable
@@ -178,4 +185,4 @@ void Grader::buildResults() {
   fillToolchainResultsJSON();
 }
 
-} // End namespace tester
+} // End namespace tester

src/config/Config.cpp

@@ -1,19 +1,23 @@
 #include "config/Config.h"
 
 #include "util.h"
+#include "Colors.h"
 
 #include "CLI11.hpp"
 
 #include "json.hpp"
 
 #include <iostream>
 
+#define WARN(msg) \
+  std::cout << Colors::YELLOW << "WARNING: " << Colors::RESET << msg << std::endl;
+
 // Convenience.
 using JSON = nlohmann::json;
 
 namespace tester {
 
-Config::Config(int argc, char** argv) : timeout(2l) {
+Config::Config(int argc, char** argv) : timeout(2l), numThreads(1), batchSize(5) {
 
   CLI::App app{"CMPUT 415 testing utility"};
 
@@ -33,7 +37,11 @@ Config::Config(int argc, char** argv) : timeout(2l) {
   app.add_flag("-t,--time", time, "Include the timings (seconds) of each test in the output.");
   app.add_flag_function("-v", [&](size_t count) { verbosity = static_cast<int>(count); },
                         "Increase verbosity level");
-
+
+  // multithreading options
+  app.add_option("-j", numThreads, "The number of threads on which to execute tests.");
+  app.add_option("--batch-size", batchSize, "(ADVANCED) the number of tests for each thread to grab per iteration. (default = 5)");
+
   // Enforce that if a grade path is supplied, then a log file should be as well and vice versa
   gradeOpt->needs(solutionFailureLogOpt);
   solutionFailureLogOpt->needs(gradeOpt);
@@ -106,6 +114,12 @@ Config::Config(int argc, char** argv) : timeout(2l) {
   for (auto it = tcJson.begin(); it != tcJson.end(); ++it) {
     toolchains.emplace(std::make_pair(it.key(), ToolChain(it.value(), timeout)));
   }
+
+  if (numThreads < 1)
+    throw std::runtime_error("Cannot execute on less than one thread.");
+
+  if (numThreads > std::thread::hardware_concurrency())
+    WARN("More threads than hardware supported concurrent threads, performance may suffer");
 }
 
-} // namespace tester
+} // namespace tester