From 9ed543bf7923db3574041c5477d024af4d0a167f Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 05:45:34 -0600
Subject: [PATCH 01/15] Added multithreading options

---
 include/config/Config.h | 11 +++++++++++
 src/config/Config.cpp   | 19 +++++++++++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/include/config/Config.h b/include/config/Config.h
index 2979d767..8502952d 100644
--- a/include/config/Config.h
+++ b/include/config/Config.h
@@ -3,6 +3,7 @@
 
 #include "toolchain/ToolChain.h"
 
+#include <cstdint>
 #include <filesystem>
 #include <map>
 #include <string>
@@ -50,13 +51,17 @@ class Config {
   bool isTimed() const { return time; }
   bool isMemoryChecked() const { return memory; }
   int getVerbosity() const { return verbosity; }
+  bool isStrict() const { return strict; }
 
   // Config int getters.
   int64_t getTimeout() const { return timeout; }
+  int64_t getNumThreads() const { return numThreads; }
+  int8_t getBatchSize() const { return batchSize; }
 
   // Initialisation verification.
   bool isInitialised() const { return initialised; }
   int getErrorCode() const { return errorCode; }
+
   
 private:
   // Option file paths.
@@ -78,10 +83,16 @@ class Config {
   // Option flags.
   bool debug, time, memory;
   int verbosity{0};
+  bool strict;
 
   // The command timeout.
   int64_t timeout;
 
+  // Number of threads on which to run tests
+  int64_t numThreads;
+  // Number of tests for each thread to grab on each run
+  int8_t batchSize;
+
   // Is the config initialised or not and an appropriate error code. This
   // could be due to asking for help or a missing config file.
   bool initialised;
diff --git a/src/config/Config.cpp b/src/config/Config.cpp
index 15221ad0..65e6e293 100644
--- a/src/config/Config.cpp
+++ b/src/config/Config.cpp
@@ -1,6 +1,7 @@
 #include "config/Config.h"
 
 #include "util.h"
+#include "Colors.h"
 
 #include "CLI11.hpp"
 
@@ -8,12 +9,15 @@
 
 #include <iostream>
 
+#define WARN(msg) \
+  std::cout << Colors::YELLOW << "WARNING: " << Colors::RESET << msg << std::endl;
+
 // Convenience.
 using JSON = nlohmann::json;
 
 namespace tester {
 
-Config::Config(int argc, char** argv) : timeout(2l) {
+Config::Config(int argc, char** argv) : strict(false), timeout(2l), numThreads(1), batchSize(5) {
 
   CLI::App app{"CMPUT 415 testing utility"};
 
@@ -33,6 +37,11 @@ Config::Config(int argc, char** argv) : timeout(2l) {
   app.add_flag("-t,--time", time, "Include the timings (seconds) of each test in the output.");
   app.add_flag_function("-v", [&](size_t count) { verbosity = static_cast<int>(count); },
                         "Increase verbosity level");
+
+  // multithreading options
+  app.add_option("-j", numThreads, "The number of threads on which to execute tests.");
+  app.add_flag("--strict", strict, "Set to strict timing mode.");
+  app.add_option("--batch-size", batchSize, "(ADVANCED) the number of tests for each thread to grab on each round of selection.");
   
   // Enforce that if a grade path is supplied, then a log file should be as well and vice versa
   gradeOpt->needs(solutionFailureLogOpt);
@@ -106,6 +115,12 @@ Config::Config(int argc, char** argv) : timeout(2l) {
   for (auto it = tcJson.begin(); it != tcJson.end(); ++it) {
     toolchains.emplace(std::make_pair(it.key(), ToolChain(it.value(), timeout)));
   }
+
+  if (numThreads < 1)
+    throw std::runtime_error("Cannot execute on less than one thread.");
+
+  if (numThreads > std::thread::hardware_concurrency())
+    WARN("More threads than hardware supported concurrent threads, performance may suffer");
 }
 
-} // namespace tester
\ No newline at end of file
+} // namespace tester

From ab0ac827eaadbe9c6ea9014717363c9fd7b679cb Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 05:51:37 -0600
Subject: [PATCH 02/15] Refactored the TestSet

TestSet now contains data on the success of the test in a pair
---
 include/testharness/TestHarness.h |  7 ++++++-
 src/analysis/Grader.cpp           |  5 +++--
 src/testharness/TestHarness.cpp   | 14 ++++++++------
 src/tests/TestRunning.cpp         |  2 +-
 4 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index ac32e329..d4e8e969 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -5,6 +5,7 @@
 #include "config/Config.h"
 #include "testharness/ResultManager.h"
 #include "tests/TestParser.h"
+#include "tests/TestResult.h"
 #include "toolchain/ToolChain.h"
 
 #include <filesystem>
@@ -17,7 +18,8 @@ namespace fs = std::filesystem;
 namespace tester {
 
 // Test hierarchy types
-typedef std::vector<std::unique_ptr<TestFile>> SubPackage;
+typedef std::pair<std::unique_ptr<TestFile>, std::reference_wrapper<std::optional<TestResult>>> TestPair;
+typedef std::vector<TestPair> SubPackage;
 typedef std::map<std::string, SubPackage> Package;
 typedef std::map<std::string, Package> TestSet;
 
@@ -54,6 +56,9 @@ class TestHarness {
 
 private:
   // The results of the tests.
+  //  NOTE we keep both a result manager and
+  //  the result in the TestSet to ensure in-ordre
+  //  printing
   ResultManager results;
 
 private:
diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp
index b9ddcefd..1f4f561f 100644
--- a/src/analysis/Grader.cpp
+++ b/src/analysis/Grader.cpp
@@ -125,7 +125,8 @@ void Grader::fillToolchainResultsJSON() {
         // attacker, tracking pass count.
         size_t passCount = 0, testCount = 0;
         for (const auto& subpackages : testSet[attacker]) {
-          for (const std::unique_ptr<TestFile>& test : subpackages.second) {
+          for (const TestPair& testpair : subpackages.second) {
+            const std::unique_ptr<TestFile>& test = testpair.first;
 
             TestResult result = runTest(test.get(), tc, cfg);
             
@@ -178,4 +179,4 @@ void Grader::buildResults() {
   fillToolchainResultsJSON();
 }
 
-} // End namespace tester
\ No newline at end of file
+} // End namespace tester
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index a6bc59c5..d56247da 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -6,6 +6,7 @@
 
 #include <filesystem>
 #include <iostream>
+#include <optional>
 #include <sstream>
 #include <utility>
 
@@ -83,7 +84,7 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
 
       // Iterate over each test in the package
       for (size_t i = 0; i < subPackage.size(); ++i) {
-        std::unique_ptr<TestFile>& test = subPackage[i];
+        std::unique_ptr<TestFile>& test = subPackage[i].first;
         if (test->getParseError() == ParseError::NoError) {
         
           TestResult result = runTest(test.get(), toolChain, cfg);
@@ -119,8 +120,8 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
             << "\n";
 
   for (auto& test : invalidTests) {
-    std::cout << "  Skipped: " << test->getTestPath().filename().stem() << std::endl
-              << "  Error: " << Colors::YELLOW << test->getParseErrorMsg() << Colors::RESET << "\n";
+    std::cout << "  Skipped: " << test.first->getTestPath().filename().stem() << std::endl
+              << "  Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n";
   }
   std::cout << "\n";
 
@@ -150,10 +151,11 @@ void TestHarness::addTestFileToSubPackage(SubPackage& subPackage, const fs::path
 
   TestParser parser(testfile.get());
 
+  std::optional<TestResult> no_result = std::nullopt;
   if (testfile->didError()) {
-    invalidTests.push_back(std::move(testfile));
-  } else {
-    subPackage.push_back(std::move(testfile));
+    invalidTests.push_back({std::move(testfile), no_result});
+  }else {
+    subPackage.push_back({std::move(testfile), no_result});
   }
 }
 
diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp
index a4ac1015..c010f412 100644
--- a/src/tests/TestRunning.cpp
+++ b/src/tests/TestRunning.cpp
@@ -245,4 +245,4 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg
   return TestResult(testPath, !testDiff, testError, "");
 }
 
-} // End namespace tester
\ No newline at end of file
+} // End namespace tester

From e6777c0fe4e4e72743ac80f078f6f946a0fb2e5f Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 06:12:16 -0600
Subject: [PATCH 03/15] Populate result information in TestPair

Put the data into the optional. This was surprisingly non-trivial

I really hate C++
---
 include/testharness/TestHarness.h |  2 +-
 include/tests/TestResult.h        | 19 ++++++++++++++-----
 src/testharness/TestHarness.cpp   |  8 ++++++++
 3 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index d4e8e969..3c2d2079 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -18,7 +18,7 @@ namespace fs = std::filesystem;
 namespace tester {
 
 // Test hierarchy types
-typedef std::pair<std::unique_ptr<TestFile>, std::reference_wrapper<std::optional<TestResult>>> TestPair;
+typedef std::pair<std::unique_ptr<TestFile>, std::optional<TestResult>> TestPair;
 typedef std::vector<TestPair> SubPackage;
 typedef std::map<std::string, SubPackage> Package;
 typedef std::map<std::string, Package> TestSet;
diff --git a/include/tests/TestResult.h b/include/tests/TestResult.h
index 62ff9c19..8a1621a1 100644
--- a/include/tests/TestResult.h
+++ b/include/tests/TestResult.h
@@ -17,12 +17,21 @@ struct TestResult {
       : name(in.stem()), pass(pass), error(error), diff(diff) {}
 
   // Info about result.
-  const fs::path name;
-  const bool pass;
-  const bool error;
-  const std::string diff;
-};
+  fs::path name;
+  bool pass;
+  bool error;
+  std::string diff;
+
+  TestResult clone() { return TestResult(this->name, this->pass, this->error, this->diff); }
 
+  void swap(TestResult &other) {
+    std::swap(name, other.name);
+    std::swap(pass, other.pass);
+    std::swap(error, other.error);
+    std::swap(diff, other.diff);
+  }
+
+};
 } // End namespace tester
 
 #endif // TESTER_TEST_RESULT_H
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index d56247da..c702d4fa 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -12,6 +12,10 @@
 
 namespace tester {
 
+void swap(TestResult& first, TestResult& second) {
+  std::swap(first, second);
+}
+
 // Builds TestSet during object creation.
 bool TestHarness::runTests() {
   bool failed = false;
@@ -88,6 +92,10 @@ bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName)
         if (test->getParseError() == ParseError::NoError) {
         
           TestResult result = runTest(test.get(), toolChain, cfg);
+          // keep the result with the test for pretty printing
+          std::optional<TestResult> res_clone = std::make_optional(result.clone());
+          subPackage[i].second.swap(res_clone);
+
           results.addResult(exeName, tcName, subPackageName, result);
           printTestResult(test.get(), result); 
 

From 71f99621b96adf1bd7e694939001717be29e82cf Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 07:02:05 -0600
Subject: [PATCH 04/15] Seperation of test running and result printing

---
 include/testharness/TestHarness.h |   2 +
 src/testharness/TestHarness.cpp   | 116 +++++++++++++++++++++++++++++-
 2 files changed, 117 insertions(+), 1 deletion(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index 3c2d2079..a77b2b1c 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -64,9 +64,11 @@ class TestHarness {
 private:
   // test running
   bool runTestsForToolChain(std::string tcId, std::string exeName);
+  void threadRunTestsForToolChain(std::string tcId, std::string exeName);
 
   // helper for formatting tester output 
   void printTestResult(const TestFile *test, TestResult result);
+  bool aggregateTestResultsForToolChain(std::string tcName, std::string exeName);
 
   // test finding and filling methods
   void addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file);
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index c702d4fa..05ccebc9 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -4,10 +4,12 @@
 #include "tests/TestRunning.h"
 #include "util.h"
 
+#include <chrono>
 #include <filesystem>
 #include <iostream>
 #include <optional>
 #include <sstream>
+#include <thread>
 #include <utility>
 
 namespace tester {
@@ -23,8 +25,11 @@ bool TestHarness::runTests() {
   for (auto exePair : cfg.getExecutables()) {
     // Iterate over toolchains.
     for (auto& tcPair : cfg.getToolChains()) {
-      if (runTestsForToolChain(exePair.first, tcPair.first) == 1)
+      std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first);
+      if (aggregateTestResultsForToolChain(tcPair.first, exePair.first) == 1)
         failed = true;
+
+      t.join();
     }
   }
   return failed;
@@ -59,6 +64,115 @@ void TestHarness::printTestResult(const TestFile *test, TestResult result) {
   std::cout << "\n";
 }
 
+bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::string exeName) {
+  bool failed = false;
+
+  ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use.
+  const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
+  toolChain.setTestedExecutable(exe);
+
+  if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well.
+    toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
+  else
+    toolChain.setTestedRuntime("");
+
+  std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
+  std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';
+
+  unsigned int toolChainCount = 0, toolChainPasses = 0; // Stat tracking for toolchain tests.
+
+  // Iterate over each package.
+  for (auto& [packageName, package] : testSet) {
+    std::cout << "Entering package: " << packageName << '\n';
+    unsigned int packageCount = 0, packagePasses = 0;
+
+    // Iterate over each subpackage
+    for (auto& [subPackageName, subPackage] : package) {
+      std::cout << "  Entering subpackage: " << subPackageName << '\n';
+      unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();
+
+      // Iterate over each test in the package
+      for (size_t i = 0; i < subPackage.size(); ++i) {
+        TestPair& pair = subPackage[i];
+        std::unique_ptr<TestFile>& test = pair.first;
+        if (test->getParseError() == ParseError::NoError) {
+
+          // Poll while we wait for the result
+          // TODO this could probably be replaced with some sort of interrupt,
+          //  (and probably should be), but better this than no threads
+          while (!pair.second.has_value())
+            std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+          TestResult result = pair.second.value();
+
+          // keep the result with the test for pretty printing
+          std::optional<TestResult> res_clone = std::make_optional(result.clone());
+          subPackage[i].second.swap(res_clone);
+
+          results.addResult(exeName, tcName, subPackageName, result);
+          printTestResult(test.get(), result);
+
+          if (result.pass) {
+            ++packagePasses;
+            ++subPackagePasses;
+          } else {
+            failed = true;
+          }
+        } else {
+          std::cout << "    " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) << " "
+                    << test->getTestPath().stem().string() << '\n';
+          --subPackageSize;
+        }
+      }
+      std::cout << "  Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n';
+      // Track how many tests we run.
+      packageCount += subPackageSize;
+    }
+
+    // Update the toolchain stats from the package stats.
+    toolChainPasses += packagePasses;
+    toolChainCount += packageCount;
+
+    std::cout << " Package passed " << packagePasses << " / " << packageCount << '\n';
+  }
+
+  std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
+  std::cout << "Invalid " << invalidTests.size() << " / " << toolChainCount + invalidTests.size()
+            << "\n";
+
+  for (auto& test : invalidTests) {
+    std::cout << "  Skipped: " << test.first->getTestPath().filename().stem() << std::endl
+              << "  Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n";
+  }
+  std::cout << "\n";
+
+  return failed;
+}
+
+void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exeName) {
+  ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use.
+  const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
+  toolChain.setTestedExecutable(exe);
+
+  // Iterate over each package.
+  for (auto& [packageName, package] : testSet) {
+    // Iterate over each subpackage
+    for (auto& [subPackageName, subPackage] : package) {
+      // Iterate over each test in the package
+      for (size_t i = 0; i < subPackage.size(); ++i) {
+        std::unique_ptr<TestFile>& test = subPackage[i].first;
+        if (test->getParseError() == ParseError::NoError) {
+
+          TestResult result = runTest(test.get(), toolChain, cfg);
+          // keep the result with the test for pretty printing
+          std::optional<TestResult> res_clone = std::make_optional(result.clone());
+          subPackage[i].second.swap(res_clone);
+        }
+      }
+    }
+  }
+}
+
 bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) {
   bool failed = false;
 

From c57ed7217a4e1c88ab2b10911f6de3f83c4a3f17 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 07:03:27 -0600
Subject: [PATCH 05/15] Naive multithreading support

Simply spawns a thread for every toolchain.

Note that this would probably brick a computer without a lot of
cores or running tests with a whole bunch of packages.

Implementation will need to limit the number of spawnable threads
in the future.
---
 src/testharness/TestHarness.cpp | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index 05ccebc9..5eb30a73 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -20,16 +20,28 @@ void swap(TestResult& first, TestResult& second) {
 
 // Builds TestSet during object creation.
 bool TestHarness::runTests() {
-  bool failed = false;
+  std::vector<std::thread> threadPool;
+
+  // Initialize the threads
   // Iterate over executables.
   for (auto exePair : cfg.getExecutables()) {
     // Iterate over toolchains.
     for (auto& tcPair : cfg.getToolChains()) {
       std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first);
+      threadPool.push_back(std::move(t));
+    }
+  }
+
+  bool failed = false;
+  // Iterate over executables.
+  for (auto exePair : cfg.getExecutables()) {
+    // Iterate over toolchains.
+    for (auto& tcPair : cfg.getToolChains()) {
       if (aggregateTestResultsForToolChain(tcPair.first, exePair.first) == 1)
         failed = true;
 
-      t.join();
+      threadPool.back().join();
+      threadPool.pop_back();
     }
   }
   return failed;

From 1a65cc62b6460a6ce36c0436a3773ace45529045 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 07:06:27 -0600
Subject: [PATCH 06/15] Remove unused function

---
 include/testharness/TestHarness.h |  1 -
 src/testharness/TestHarness.cpp   | 77 -------------------------------
 2 files changed, 78 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index a77b2b1c..a66e790c 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -63,7 +63,6 @@ class TestHarness {
 
 private:
   // test running
-  bool runTestsForToolChain(std::string tcId, std::string exeName);
   void threadRunTestsForToolChain(std::string tcId, std::string exeName);
 
   // helper for formatting tester output 
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index 5eb30a73..a0d2bac0 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -185,83 +185,6 @@ void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exe
   }
 }
 
-bool TestHarness::runTestsForToolChain(std::string exeName, std::string tcName) {
-  bool failed = false;
-
-  ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use.
-  const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
-  toolChain.setTestedExecutable(exe);
-
-  if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well.
-    toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
-  else
-    toolChain.setTestedRuntime("");
-
-  std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
-  std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';
-
-  unsigned int toolChainCount = 0, toolChainPasses = 0; // Stat tracking for toolchain tests.
-
-  // Iterate over each package.
-  for (auto& [packageName, package] : testSet) {
-    std::cout << "Entering package: " << packageName << '\n';
-    unsigned int packageCount = 0, packagePasses = 0;
-
-    // Iterate over each subpackage
-    for (auto& [subPackageName, subPackage] : package) {
-      std::cout << "  Entering subpackage: " << subPackageName << '\n';
-      unsigned int subPackagePasses = 0, subPackageSize = subPackage.size();
-
-      // Iterate over each test in the package
-      for (size_t i = 0; i < subPackage.size(); ++i) {
-        std::unique_ptr<TestFile>& test = subPackage[i].first;
-        if (test->getParseError() == ParseError::NoError) {
-        
-          TestResult result = runTest(test.get(), toolChain, cfg);
-          // keep the result with the test for pretty printing
-          std::optional<TestResult> res_clone = std::make_optional(result.clone());
-          subPackage[i].second.swap(res_clone);
-
-          results.addResult(exeName, tcName, subPackageName, result);
-          printTestResult(test.get(), result); 
-
-          if (result.pass) {
-            ++packagePasses;
-            ++subPackagePasses;
-          } else {
-            failed = true;
-          }
-        } else {
-          std::cout << "    " << (Colors::YELLOW + "[INVALID]" + Colors::RESET) << " "
-                    << test->getTestPath().stem().string() << '\n';
-          --subPackageSize;
-        }        
-      }
-      std::cout << "  Subpackage passed " << subPackagePasses << " / " << subPackageSize << '\n';
-      // Track how many tests we run.
-      packageCount += subPackageSize;
-    }
-
-    // Update the toolchain stats from the package stats.
-    toolChainPasses += packagePasses;
-    toolChainCount += packageCount;
-
-    std::cout << " Package passed " << packagePasses << " / " << packageCount << '\n';
-  }
-
-  std::cout << "Toolchain passed " << toolChainPasses << " / " << toolChainCount << "\n\n";
-  std::cout << "Invalid " << invalidTests.size() << " / " << toolChainCount + invalidTests.size()
-            << "\n";
-
-  for (auto& test : invalidTests) {
-    std::cout << "  Skipped: " << test.first->getTestPath().filename().stem() << std::endl
-              << "  Error: " << Colors::YELLOW << test.first->getParseErrorMsg() << Colors::RESET << "\n";
-  }
-  std::cout << "\n";
-
-  return failed;
-}
-
 bool isTestFile(const fs::path& path) {
   return (fs::exists(path) 
       && !fs::is_directory(path)

From e1ef0a17b45145de8a279cc67e66d1de0aff96b2 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 07:25:23 -0600
Subject: [PATCH 07/15] Implemented numThread limiting

Currently there is a delay when we try to kill the threads, I'm
not entirely sure why, but it makes it seem like the process takes
a long time to die. Something to investigate in the future, but
threading is fully implemented.
---
 include/testharness/TestHarness.h |  3 +++
 src/testharness/TestHarness.cpp   | 45 ++++++++++++++++++++++++-------
 2 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index a66e790c..3a741f42 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -62,6 +62,9 @@ class TestHarness {
   ResultManager results;
 
 private:
+  // thread control
+  void spawnThreads();
+
   // test running
   void threadRunTestsForToolChain(std::string tcId, std::string exeName);
 
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index a0d2bac0..7102ed97 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -20,31 +20,53 @@ void swap(TestResult& first, TestResult& second) {
 
 // Builds TestSet during object creation.
 bool TestHarness::runTests() {
-  std::vector<std::thread> threadPool;
+  // initialize the threads
+  std::thread t(&TestHarness::spawnThreads, this);
 
-  // Initialize the threads
+  bool failed = false;
   // Iterate over executables.
   for (auto exePair : cfg.getExecutables()) {
     // Iterate over toolchains.
     for (auto& tcPair : cfg.getToolChains()) {
-      std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first);
-      threadPool.push_back(std::move(t));
+      if (aggregateTestResultsForToolChain(tcPair.first, exePair.first) == 1)
+        failed = true;
     }
   }
 
-  bool failed = false;
+  // join the control thread
+  t.join();
+  return failed;
+}
+
+void TestHarness::spawnThreads() {
+  int16_t numThreads = 0;
+  std::vector<std::thread> threadPool;
+
+  // Initialize the threads
   // Iterate over executables.
   for (auto exePair : cfg.getExecutables()) {
     // Iterate over toolchains.
     for (auto& tcPair : cfg.getToolChains()) {
-      if (aggregateTestResultsForToolChain(tcPair.first, exePair.first) == 1)
-        failed = true;
+      // If we have already spawned the maximum number of threads,
+      //  wait for the first one to finish before spawning another.
+      if (numThreads >= cfg.getNumThreads()) {
+        threadPool.back().join();
+        threadPool.pop_back();
+        numThreads--;
+      }
 
-      threadPool.back().join();
-      threadPool.pop_back();
+      // spawn a new thread executing its tests
+      std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first);
+      threadPool.push_back(std::move(t));
+      numThreads++;
     }
   }
-  return failed;
+
+  // Join any stragglers
+  assert(threadPool.size() <= cfg.getNumThreads());
+  for (size_t i = 0; i < threadPool.size(); i++) {
+    threadPool[i].join();
+  }
 }
 
 std::string TestHarness::getTestInfo() const {
@@ -158,6 +180,9 @@ bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::stri
   }
   std::cout << "\n";
 
+  std::cout << Colors::GREEN << "Completed Tests" << Colors::RESET << std::endl;
+  std::cout << "Hold on while we clean up any remaining threads, this might take a moment" << std::endl;
+
   return failed;
 }
 

From 10ab31fb007436d9da30bd6a7d6b6f1ce4e23f41 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 07:31:47 -0600
Subject: [PATCH 08/15] Removed batch size and strict options

**Batch Size**

Not necessary since the threads will just grab everything in their
subpackage. In the future it would be nice to, instead of
seperating based on packages:
1. Generate the set of all tests
2. Threads access batchSize tests at a time
3. Threads then keep referring back to the test pool for which
    tests need executing

This would eliminate the need for constantly restarting threads,
but would take a bit of a refactor to implement correctly.

**Strict**

I'm tired. I hate writing C++. I'm going to sleep. Here's my PR.
---
 include/config/Config.h | 4 ----
 src/config/Config.cpp   | 6 ++----
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/include/config/Config.h b/include/config/Config.h
index 8502952d..be6622f5 100644
--- a/include/config/Config.h
+++ b/include/config/Config.h
@@ -51,12 +51,10 @@ class Config {
   bool isTimed() const { return time; }
   bool isMemoryChecked() const { return memory; }
   int getVerbosity() const { return verbosity; }
-  bool isStrict() const { return strict; }
 
   // Config int getters.
   int64_t getTimeout() const { return timeout; }
   int64_t getNumThreads() const { return numThreads; }
-  int8_t getBatchSize() const { return batchSize; }
 
   // Initialisation verification.
   bool isInitialised() const { return initialised; }
@@ -83,7 +81,6 @@ class Config {
   // Option flags.
   bool debug, time, memory;
   int verbosity{0};
-  bool strict;
 
   // The command timeout.
   int64_t timeout;
@@ -91,7 +88,6 @@ class Config {
   // Number of threads on which to run tests
   int64_t numThreads;
   // Number of tests for each thread to grab on each run
-  int8_t batchSize;
 
   // Is the config initialised or not and an appropriate error code. This
   // could be due to asking for help or a missing config file.
diff --git a/src/config/Config.cpp b/src/config/Config.cpp
index 65e6e293..7cad2469 100644
--- a/src/config/Config.cpp
+++ b/src/config/Config.cpp
@@ -17,7 +17,7 @@ using JSON = nlohmann::json;
 
 namespace tester {
 
-Config::Config(int argc, char** argv) : strict(false), timeout(2l), numThreads(1), batchSize(5) {
+Config::Config(int argc, char** argv) : timeout(2l), numThreads(1) {
 
   CLI::App app{"CMPUT 415 testing utility"};
 
@@ -40,9 +40,7 @@ Config::Config(int argc, char** argv) : strict(false), timeout(2l), numThreads(1
 
   // multithreading options
   app.add_option("-j", numThreads, "The number of threads on which to execute tests.");
-  app.add_flag("--strict", strict, "Set to strict timing mode.");
-  app.add_option("--batch-size", batchSize, "(ADVANCED) the number of tests for each thread to grab on each round of selection.");
-  
+
   // Enforce that if a grade path is supplied, then a log file should be as well and vice versa
   gradeOpt->needs(solutionFailureLogOpt);
   solutionFailureLogOpt->needs(gradeOpt);

From 723a411475265db79c2b528388e5a2bee6d4a4b1 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 08:02:29 -0600
Subject: [PATCH 09/15] Fixed comparison warning

---
 src/testharness/TestHarness.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index 7102ed97..8dad433b 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -63,7 +63,7 @@ void TestHarness::spawnThreads() {
   }
 
   // Join any stragglers
-  assert(threadPool.size() <= cfg.getNumThreads());
+  assert(threadPool.size() <= (size_t) cfg.getNumThreads());
   for (size_t i = 0; i < threadPool.size(); i++) {
     threadPool[i].join();
   }

From f518f6c608ae3a029e762e30f2efbaf00df9c2cd Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 08:11:35 -0600
Subject: [PATCH 10/15] Fixed Runtimes

Forgot to move it from the printing function to the execution
one.
---
 src/testharness/TestHarness.cpp | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index 8dad433b..4b9bbde2 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -105,11 +105,6 @@ bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::stri
   const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
   toolChain.setTestedExecutable(exe);
 
-  if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well.
-    toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
-  else
-    toolChain.setTestedRuntime("");
-
   std::cout << "\nTesting executable: " << exeName << " -> " << exe << '\n';
   std::cout << "With toolchain: " << tcName << " -> " << toolChain.getBriefDescription() << '\n';
 
@@ -191,6 +186,13 @@ void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exe
   const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
   toolChain.setTestedExecutable(exe);
 
+  // set the runtime
+  if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well.
+    toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
+  else
+    toolChain.setTestedRuntime("");
+
+
   // Iterate over each package.
   for (auto& [packageName, package] : testSet) {
     // Iterate over each subpackage

From 66e52d15617c825917cb35128979c0c1cc180f65 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 15:14:21 -0600
Subject: [PATCH 11/15] Fixed granularity of multithreading

Previously, the threads were individually executing an entire
toolchain on their own, switched that to actually do the subpackage
like I claimed it did.
---
 include/testharness/TestHarness.h |  2 +-
 src/analysis/Grader.cpp           |  1 +
 src/testharness/TestHarness.cpp   | 53 +++++++++++++++----------------
 3 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index 3a741f42..2cc304de 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -66,7 +66,7 @@ class TestHarness {
   void spawnThreads();
 
   // test running
-  void threadRunTestsForToolChain(std::string tcId, std::string exeName);
+  void threadRunTestsForToolChain(std::string tcId, std::string exeName, std::reference_wrapper<SubPackage> subpackage);
 
   // helper for formatting tester output 
   void printTestResult(const TestFile *test, TestResult result);
diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp
index 1f4f561f..1c0bd344 100644
--- a/src/analysis/Grader.cpp
+++ b/src/analysis/Grader.cpp
@@ -1,6 +1,7 @@
 #include "analysis/Grader.h"
 #include <algorithm>
 #include <iostream>
+#include <thread>
 
 namespace {
 
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index 4b9bbde2..9607e1c9 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -47,18 +47,24 @@ void TestHarness::spawnThreads() {
   for (auto exePair : cfg.getExecutables()) {
     // Iterate over toolchains.
     for (auto& tcPair : cfg.getToolChains()) {
-      // If we have already spawned the maximum number of threads,
-      //  wait for the first one to finish before spawning another.
-      if (numThreads >= cfg.getNumThreads()) {
-        threadPool.back().join();
-        threadPool.pop_back();
-        numThreads--;
-      }
+      // iterate over packages
+      for (auto& package : testSet) { // TestSet.second -> Package
+        // iterate over subpackages
+        for (auto& subpackage : package.second) { // Package.second -> SubPackage
+          // If we have already spawned the maximum number of threads,
+          //  wait for the first one to finish before spawning another.
+          if (numThreads >= cfg.getNumThreads()) {
+            threadPool.back().join();
+            threadPool.pop_back();
+            numThreads--;
+          }
 
-      // spawn a new thread executing its tests
-      std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first);
-      threadPool.push_back(std::move(t));
-      numThreads++;
+          // spawn a new thread executing its tests
+          std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first, std::ref(subpackage.second));
+          threadPool.push_back(std::move(t));
+          numThreads++;
+        }
+      }
     }
   }
 
@@ -181,7 +187,8 @@ bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::stri
   return failed;
 }
 
-void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exeName) {
+void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exeName, std::reference_wrapper<SubPackage> subPackage) {
+  std::cout << "Thread " << std::this_thread::get_id() << ": executing subpackage" << std::endl;
   ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use.
   const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
   toolChain.setTestedExecutable(exe);
@@ -192,22 +199,14 @@ void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exe
   else
     toolChain.setTestedRuntime("");
 
+  for (size_t i = 0; i < subPackage.get().size(); ++i) {
+    std::unique_ptr<TestFile>& test = subPackage.get().at(i).first;
+    if (test->getParseError() == ParseError::NoError) {
 
-  // Iterate over each package.
-  for (auto& [packageName, package] : testSet) {
-    // Iterate over each subpackage
-    for (auto& [subPackageName, subPackage] : package) {
-      // Iterate over each test in the package
-      for (size_t i = 0; i < subPackage.size(); ++i) {
-        std::unique_ptr<TestFile>& test = subPackage[i].first;
-        if (test->getParseError() == ParseError::NoError) {
-
-          TestResult result = runTest(test.get(), toolChain, cfg);
-          // keep the result with the test for pretty printing
-          std::optional<TestResult> res_clone = std::make_optional(result.clone());
-          subPackage[i].second.swap(res_clone);
-        }
-      }
+      TestResult result = runTest(test.get(), toolChain, cfg);
+      // keep the result with the test for pretty printing
+      std::optional<TestResult> res_clone = std::make_optional(result.clone());
+      subPackage.get().at(i).second.swap(res_clone);
     }
   }
 }

From 6594a085f8617c6042bbff952927cde9ba1ece26 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 15:15:36 -0600
Subject: [PATCH 12/15] Grader parity with TestHarness

Ensured that the grader does not try to execute everything in a
single thread, waiting for the tests to finish asynchronously.
---
 include/analysis/Grader.h | 1 -
 src/analysis/Grader.cpp   | 9 +++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/include/analysis/Grader.h b/include/analysis/Grader.h
index 3a166303..75b47345 100644
--- a/include/analysis/Grader.h
+++ b/include/analysis/Grader.h
@@ -24,7 +24,6 @@ class Grader : public TestHarness {
   Grader(const Config& cfg) : TestHarness(cfg), 
                               failedTestLog(*cfg.getFailureLogPath()),
                               solutionExecutable(*cfg.getSolutionExecutable()) {
-    findTests();
     buildResults();
   }
 
diff --git a/src/analysis/Grader.cpp b/src/analysis/Grader.cpp
index 1c0bd344..e32d3911 100644
--- a/src/analysis/Grader.cpp
+++ b/src/analysis/Grader.cpp
@@ -128,9 +128,14 @@ void Grader::fillToolchainResultsJSON() {
         for (const auto& subpackages : testSet[attacker]) {
           for (const TestPair& testpair : subpackages.second) {
             const std::unique_ptr<TestFile>& test = testpair.first;
+            // Poll while we wait for the result
+            // TODO this could probably be replaced with some sort of interrupt,
+            //  (and probably should be), but better this than no threads
+            while (!testpair.second.has_value())
+              std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+            TestResult result = testpair.second.value();
 
-            TestResult result = runTest(test.get(), tc, cfg);
-            
             if (!result.pass && defender == solutionExecutable) {
               if ( attacker == solutionExecutable ) {
                 // A testcase just failed the solution executable

From 2e44d29608c8256dfe92a1c174699b0f857d15eb Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Sun, 8 Sep 2024 18:08:43 -0600
Subject: [PATCH 13/15] Re added the batch size argument

In preparation for the use of the set generation tactic I mentioned
for better load balancing in the children
---
 include/config/Config.h | 2 ++
 src/config/Config.cpp   | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/include/config/Config.h b/include/config/Config.h
index be6622f5..6097e983 100644
--- a/include/config/Config.h
+++ b/include/config/Config.h
@@ -55,6 +55,7 @@ class Config {
   // Config int getters.
   int64_t getTimeout() const { return timeout; }
   int64_t getNumThreads() const { return numThreads; }
+  int8_t getBatchSize() const { return batchSize; }
 
   // Initialisation verification.
   bool isInitialised() const { return initialised; }
@@ -88,6 +89,7 @@ class Config {
   // Number of threads on which to run tests
   int64_t numThreads;
   // Number of tests for each thread to grab on each run
+  int8_t batchSize;
 
   // Is the config initialised or not and an appropriate error code. This
   // could be due to asking for help or a missing config file.
diff --git a/src/config/Config.cpp b/src/config/Config.cpp
index 7cad2469..c5051e45 100644
--- a/src/config/Config.cpp
+++ b/src/config/Config.cpp
@@ -17,7 +17,7 @@ using JSON = nlohmann::json;
 
 namespace tester {
 
-Config::Config(int argc, char** argv) : timeout(2l), numThreads(1) {
+Config::Config(int argc, char** argv) : timeout(2l), numThreads(1), batchSize(5) {
 
   CLI::App app{"CMPUT 415 testing utility"};
 
@@ -40,6 +40,7 @@ Config::Config(int argc, char** argv) : timeout(2l), numThreads(1) {
 
   // multithreading options
   app.add_option("-j", numThreads, "The number of threads on which to execute tests.");
+  app.add_option("--batch-size", batchSize, "(ADVANCED) the number of tests for each thread to grab per iteration. (default = 5)");
 
   // Enforce that if a grade path is supplied, then a log file should be as well and vice versa
   gradeOpt->needs(solutionFailureLogOpt);

From 3afde6a5fbae1531bd335343446a827020b838d5 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Thu, 12 Sep 2024 22:32:20 -0600
Subject: [PATCH 14/15] Multithreading works as expected

Had an indexing error due to a malformed ternary, all works now
---
 include/testharness/TestHarness.h |   9 ++-
 src/testharness/TestHarness.cpp   | 124 ++++++++++++++++++++----------
 2 files changed, 92 insertions(+), 41 deletions(-)

diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h
index 2cc304de..3c849fa8 100644
--- a/include/testharness/TestHarness.h
+++ b/include/testharness/TestHarness.h
@@ -51,6 +51,7 @@ class TestHarness {
   // A separate subpackage, just for invalid tests.
   SubPackage invalidTests;
 
+protected:
   // let derived classes find tests.
   void findTests();
 
@@ -66,7 +67,13 @@ class TestHarness {
   void spawnThreads();
 
   // test running
-  void threadRunTestsForToolChain(std::string tcId, std::string exeName, std::reference_wrapper<SubPackage> subpackage);
+  void threadRunTestBatch(std::reference_wrapper<std::vector<std::string>> toolchains,
+                          std::reference_wrapper<std::vector<std::string>> executables,
+                          std::reference_wrapper<std::vector<std::reference_wrapper<TestPair>>> tests,
+                          std::reference_wrapper<size_t> currentIndex, std::reference_wrapper<std::mutex> lock);
+  void threadRunTestsForToolChain(std::reference_wrapper<std::vector<std::string>> tcIds,
+                                  std::reference_wrapper<std::vector<std::string>> exeNames,
+                                  std::reference_wrapper<std::vector<std::reference_wrapper<TestPair>>> tests, size_t begin, size_t end);
 
   // helper for formatting tester output 
   void printTestResult(const TestFile *test, TestResult result);
diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index 9607e1c9..f65f02c1 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -7,6 +7,7 @@
 #include <chrono>
 #include <filesystem>
 #include <iostream>
+#include <mutex>
 #include <optional>
 #include <sstream>
 #include <thread>
@@ -39,8 +40,13 @@ bool TestHarness::runTests() {
 }
 
 void TestHarness::spawnThreads() {
-  int16_t numThreads = 0;
   std::vector<std::thread> threadPool;
+  std::vector<std::reference_wrapper<TestPair>> flattenedList;
+  std::vector<std::string> exeList;
+  std::vector<std::string> tcList;
+
+  static size_t currentIndex = 0;
+  std::mutex currentIndexLock;
 
   // Initialize the threads
   // Iterate over executables.
@@ -51,27 +57,89 @@ void TestHarness::spawnThreads() {
       for (auto& package : testSet) { // TestSet.second -> Package
         // iterate over subpackages
         for (auto& subpackage : package.second) { // Package.second -> SubPackage
-          // If we have already spawned the maximum number of threads,
-          //  wait for the first one to finish before spawning another.
-          if (numThreads >= cfg.getNumThreads()) {
-            threadPool.back().join();
-            threadPool.pop_back();
-            numThreads--;
+          // populate the flattened test vector
+          for (auto& test : subpackage.second) {
+            flattenedList.push_back(std::ref(test));
+            exeList.push_back(exePair.first);
+            tcList.push_back(tcPair.first);
           }
-
-          // spawn a new thread executing its tests
-          std::thread t(&TestHarness::threadRunTestsForToolChain, this, tcPair.first, exePair.first, std::ref(subpackage.second));
-          threadPool.push_back(std::move(t));
-          numThreads++;
         }
       }
     }
   }
 
-  // Join any stragglers
-  assert(threadPool.size() <= (size_t) cfg.getNumThreads());
-  for (size_t i = 0; i < threadPool.size(); i++) {
-    threadPool[i].join();
+  // Let the load balancing be the responsability of the threads, instead of the supervisor
+  for (int64_t i = 0; i < cfg.getNumThreads(); i++) {
+    std::thread t(&TestHarness::threadRunTestBatch, this,
+                  std::ref(tcList),
+                  std::ref(exeList),
+                  std::ref(flattenedList),
+                  std::ref(currentIndex),
+                  std::ref(currentIndexLock));
+    threadPool.push_back(std::move(t));
+  }
+
+  // KILL ALL THE CHILDREN
+  for (auto& thread : threadPool) thread.join();
+}
+
+void TestHarness::threadRunTestBatch(std::reference_wrapper<std::vector<std::string>> toolchains,
+                        std::reference_wrapper<std::vector<std::string>> executables,
+                        std::reference_wrapper<std::vector<std::reference_wrapper<TestPair>>> tests,
+                        std::reference_wrapper<size_t> currentIndex,
+                        std::reference_wrapper<std::mutex> currentIndexLock)
+{
+  // Loop while we have not exhausted the available tests
+  while (currentIndex < tests.get().size()) {
+    std::cout << "Entering loop" << std::endl;
+    size_t tmpIndex;
+
+    {
+      // Lock the index
+      std::lock_guard<std::mutex> lock(currentIndexLock);
+      std::cout << "Mutex Locked" << std::endl;
+
+      // store the current index
+      tmpIndex = currentIndex;
+      // increment for the next lad
+      currentIndex += cfg.getBatchSize();
+    }
+    std::cout << "Mutex Unlocked" << std::endl;
+    std::cout << "Current index: " << currentIndex << " || Tests size: " << tests.get().size() << std::endl;
+
+    size_t endIndex = ((tmpIndex + cfg.getBatchSize()) >= tests.get().size()) ?  tests.get().size() : tmpIndex + cfg.getBatchSize();
+
+    std::cout << "Thread " << std::this_thread::get_id() << ": Executing tests [" << tmpIndex << ", " << endIndex << "]" << std::endl;
+
+    threadRunTestsForToolChain(std::ref(toolchains), std::ref(executables), std::ref(tests), tmpIndex, endIndex);
+  }
+}
+
+void TestHarness::threadRunTestsForToolChain(std::reference_wrapper<std::vector<std::string>> tcNames,
+                                             std::reference_wrapper<std::vector<std::string>> exeNames,
+                                             std::reference_wrapper<std::vector<std::reference_wrapper<TestPair>>> tests,
+                                             size_t begin, size_t end)
+{
+  for (size_t i = begin; i < end; i++) {
+    std::cout << "Executing test #" << i << ": " << tests.get().at(i).get().first->getInsPath() << std::endl;
+    ToolChain toolChain = cfg.getToolChain(tcNames.get().at(i)); // Get the toolchain to use.
+    const fs::path& exe = cfg.getExecutablePath(exeNames.get().at(i)); // Set the toolchain's exe to be tested.
+    toolChain.setTestedExecutable(exe);
+
+    // set the runtime
+    if (cfg.hasRuntime(exeNames.get().at(i))) // If we have a runtime, set that as well.
+      toolChain.setTestedRuntime(cfg.getRuntimePath(exeNames.get().at(i)));
+    else
+      toolChain.setTestedRuntime("");
+
+    std::unique_ptr<TestFile>& test = tests.get().at(i).get().first;
+    if (test->getParseError() == ParseError::NoError) {
+
+      TestResult result = runTest(test.get(), toolChain, cfg);
+      // keep the result with the test for pretty printing
+      std::optional<TestResult> res_clone = std::make_optional(result.clone());
+      tests.get().at(i).get().second.swap(res_clone);
+    }
   }
 }
 
@@ -187,30 +255,6 @@ bool TestHarness::aggregateTestResultsForToolChain(std::string tcName, std::stri
   return failed;
 }
 
-void TestHarness::threadRunTestsForToolChain(std::string tcName, std::string exeName, std::reference_wrapper<SubPackage> subPackage) {
-  std::cout << "Thread " << std::this_thread::get_id() << ": executing subpackage" << std::endl;
-  ToolChain toolChain = cfg.getToolChain(tcName); // Get the toolchain to use.
-  const fs::path& exe = cfg.getExecutablePath(exeName); // Set the toolchain's exe to be tested.
-  toolChain.setTestedExecutable(exe);
-
-  // set the runtime
-  if (cfg.hasRuntime(exeName)) // If we have a runtime, set that as well.
-    toolChain.setTestedRuntime(cfg.getRuntimePath(exeName));
-  else
-    toolChain.setTestedRuntime("");
-
-  for (size_t i = 0; i < subPackage.get().size(); ++i) {
-    std::unique_ptr<TestFile>& test = subPackage.get().at(i).first;
-    if (test->getParseError() == ParseError::NoError) {
-
-      TestResult result = runTest(test.get(), toolChain, cfg);
-      // keep the result with the test for pretty printing
-      std::optional<TestResult> res_clone = std::make_optional(result.clone());
-      subPackage.get().at(i).second.swap(res_clone);
-    }
-  }
-}
-
 bool isTestFile(const fs::path& path) {
   return (fs::exists(path) 
       && !fs::is_directory(path)

From ccf8cfd8ca0780693914c475df6dddad05028b88 Mon Sep 17 00:00:00 2001
From: Ayrton Chilibeck <aa@chilibeck.com>
Date: Thu, 12 Sep 2024 22:36:41 -0600
Subject: [PATCH 15/15] Removed MT Debug info

Now just to make the actual printing happen in the proper sequence
---
 src/testharness/TestHarness.cpp | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp
index f65f02c1..1ec8c217 100644
--- a/src/testharness/TestHarness.cpp
+++ b/src/testharness/TestHarness.cpp
@@ -91,26 +91,19 @@ void TestHarness::threadRunTestBatch(std::reference_wrapper<std::vector<std::str
 {
   // Loop while we have not exhausted the available tests
   while (currentIndex < tests.get().size()) {
-    std::cout << "Entering loop" << std::endl;
     size_t tmpIndex;
 
     {
       // Lock the index
       std::lock_guard<std::mutex> lock(currentIndexLock);
-      std::cout << "Mutex Locked" << std::endl;
 
       // store the current index
       tmpIndex = currentIndex;
       // increment for the next lad
       currentIndex += cfg.getBatchSize();
     }
-    std::cout << "Mutex Unlocked" << std::endl;
-    std::cout << "Current index: " << currentIndex << " || Tests size: " << tests.get().size() << std::endl;
-
     size_t endIndex = ((tmpIndex + cfg.getBatchSize()) >= tests.get().size()) ?  tests.get().size() : tmpIndex + cfg.getBatchSize();
 
-    std::cout << "Thread " << std::this_thread::get_id() << ": Executing tests [" << tmpIndex << ", " << endIndex << "]" << std::endl;
-
     threadRunTestsForToolChain(std::ref(toolchains), std::ref(executables), std::ref(tests), tmpIndex, endIndex);
   }
 }
@@ -121,7 +114,6 @@ void TestHarness::threadRunTestsForToolChain(std::reference_wrapper<std::vector<
                                              size_t begin, size_t end)
 {
   for (size_t i = begin; i < end; i++) {
-    std::cout << "Executing test #" << i << ": " << tests.get().at(i).get().first->getInsPath() << std::endl;
     ToolChain toolChain = cfg.getToolChain(tcNames.get().at(i)); // Get the toolchain to use.
     const fs::path& exe = cfg.getExecutablePath(exeNames.get().at(i)); // Set the toolchain's exe to be tested.
     toolChain.setTestedExecutable(exe);