Create a private tmp directory for files in the config directory

include/config/Config.h

@@ -28,6 +28,9 @@ class Config {
   const std::optional<fs::path>& getDebugPath() const { return debugPackage; }
   const std::optional<fs::path>& getFailureLogPath() const { return failureLogPath; };
 
+  // Return the path which the JSON config lives in
+  const fs::path&getConfigDirPath() const { return configDirPath; }
+
   // Non optional config variables 
   const fs::path&getTestDirPath() const { return testDirPath; }
 
@@ -64,7 +67,7 @@ class Config {
   std::optional<fs::path> failureLogPath;
   std::optional<fs::path> debugPackage;
 
-  fs::path testDirPath;
+  fs::path testDirPath, configDirPath;
 
   // Option file maps.
   PathMap executables;

include/testharness/TestHarness.h

@@ -28,7 +28,17 @@ class TestHarness {
   TestHarness() = delete;
 
   // Construct the Tester with a parsed JSON file.
-  TestHarness(const Config& cfg) : cfg(cfg), results() { findTests(); }
+  TestHarness(const Config& cfg) 
+    : cfg(cfg), 
+      results()
+  {
+    // Create temporary dir for test and toolchain files
+    testArtifactsPath = fs::path(cfg.getConfigDirPath() / ".test-artifacts");
+    fs::create_directory(testArtifactsPath);
+
+    // Find tests
+    findTests();
+  } 
 
   // Returns true if any tests failed, false otherwise.
   bool runTests();
@@ -52,6 +62,9 @@ class TestHarness {
   // let derived classes find tests.
   void findTests();
 
+  // Create a local tmp path for ephemeral test input, output and toolchain files 
+  fs::path testArtifactsPath;
+
 private:
   // The results of the tests.
   ResultManager results;

include/tests/TestFile.h

@@ -7,6 +7,8 @@
 #include <iostream>
 #include <string>
 #include <vector>
+#include <optional>
+#include <atomic>
 
 namespace fs = std::filesystem;
 
@@ -25,42 +27,46 @@ class TestParser;
 class TestFile {
 public:
   TestFile() = delete;
-
   // construct Testfile from path to .test file.
-  TestFile(const fs::path& path);
+  TestFile(const fs::path& path, const fs::path& artifactDir);
   ~TestFile();
 
-  uint64_t id;
+  uint64_t getId() const { return id; }
 
-  // getters
-  fs::path getTestPath() const { return testPath; }
-  fs::path getInsPath() const { return insPath; }
-  fs::path getOutPath() const { return outPath; }
+  // Test path getters
+  const fs::path& getTestPath() const { return testPath; }
+  const fs::path& getOutPath() const { return outPath; }
+  const fs::path& getInsPath() const { return insPath; }
+
+  // Test state getters 
   ParseError getParseError() const { return errorState; }
   std::string getParseErrorMsg() const;
   double getElapsedTime() const { return elapsedTime; }
   bool didError() const { return errorState != ParseError::NoError; }
 
-  // setters
-  void setTestPath(fs::path path) { testPath = path; }
+  // Test path getters
   void setInsPath(fs::path path) { insPath = path; }
   void setOutPath(fs::path path) { outPath = path; }
-  void getParseError(ParseError error) { errorState = error; }
+
+  // Test path setters
+  void setParseError(ParseError error) { errorState = error; }
   void setParseErrorMsg(std::string msg) { errorMsg = msg; }
   void setElapsedTime(double elapsed) { elapsedTime = elapsed; }
 
-  // if test has any input and if test uses input file specifically
-  bool usesInputStream{false}, usesInputFile{false}; 
-  bool usesOutStream{false}, usesOutFile{false};
-
   friend class TestParser;
 
+private:
+ static uint64_t generateId(); 
+
 protected:
-  static uint64_t nextId;
+  static std::atomic<uint64_t> nextId;
 
 private:
+  uint64_t id;
   // Path for the test, ins and out files 
-  fs::path testPath, insPath, outPath;
+  fs::path testPath;
+  fs::path outPath;
+  fs::path insPath;
 
   // Test file breaks some convention or was unable to parse directives.
   ParseError errorState{ParseError::NoError};

include/tests/TestParser.h

@@ -5,6 +5,7 @@
 #include "TestFile.h"
 #include <filesystem>
 #include <variant>
+#include <sstream>
 
 namespace fs = std::filesystem;
 
@@ -42,8 +43,7 @@ class TestParser {
   // helper method to insert a newline prefixed line to a file
   void insLineToFile(fs::path filePath, std::string line, bool firstInsert);
 
-  // methods below look for INPUT, CHECK, INPUT_FILE, CHECK_FILE directive in
-  // a lines
+  // identifiy for INPUT, CHECK, INPUT_FILE or CHECK_FILE directive in a line
   ParseError matchInputDirective(std::string& line);
   ParseError matchCheckDirective(std::string& line);
   ParseError matchInputFileDirective(std::string& line);

include/toolchain/Command.h

@@ -32,7 +32,7 @@ class Command {
   Command(const Command& command) = default;
 
   // Destructor for removing temporary files
-  ~Command() {}
+  ~Command();
 
   // Execute the command.
   ExecutionOutput execute(const ExecutionInput& ei) const;
@@ -58,15 +58,13 @@ class Command {
   std::string name;
   fs::path exePath;
   fs::path runtimePath;
+  fs::path tmpPath;
   std::vector<std::string> args;
-
+  
   // Every command produces a file descriptor for each of these paths
   fs::path errPath;
   fs::path outPath;
 
-  // The command can supply a output file to use instead of stdout/err 
-  std::optional<fs::path> outputFile;
-
   // Uses runtime and uses input stream.
   bool usesRuntime, usesInStr;
 

include/toolchain/ExecutionState.h

@@ -1,6 +1,7 @@
 #ifndef TESTER_EXECUTION_STATE_H
 #define TESTER_EXECUTION_STATE_H
 
+#include <iostream>
 #include <filesystem>
 #include <optional>
 namespace fs = std::filesystem;
@@ -16,8 +17,14 @@ class ExecutionInput {
   // Creates input to a subprocess execution.
   ExecutionInput(fs::path inputPath, fs::path inputStreamPath, fs::path testedExecutable,
                  fs::path testedRuntime)
-      : inputPath(std::move(inputPath)), inputStreamPath(std::move(inputStreamPath)),
-        testedExecutable(std::move(testedExecutable)), testedRuntime(std::move(testedRuntime)) {}
+      : inputPath(std::move(inputPath)),
+        inputStreamPath(std::move(inputStreamPath)),
+        testedExecutable(std::move(testedExecutable)),
+        testedRuntime(std::move(testedRuntime)) {}
+
+  ~ExecutionInput() {
+    // std::cout << "Destory execution input: " << inputPath << std::endl;
+  }
 
   // Gets input file.
   const fs::path& getInputFile() const { return inputPath; }
@@ -50,6 +57,10 @@ class ExecutionOutput {
         errPath(std::move(errPath)), 
         rv(0), elapsedTime(0), hasElapsed(false), isErrorTest(false) {}
 
+  ~ExecutionOutput() {
+    // std::cout << "Destory execution output: " << outPath << std::endl;
+  }
+
   // Gets output file.
   fs::path getOutputFile() const { return outPath; }
   fs::path getErrorFile() const { return errPath; }

src/config/Config.cpp

@@ -55,6 +55,11 @@ Config::Config(int argc, char** argv) : timeout(2l) {
   JSON json;
   jsonFile >> json;
 
+  // Set the config path from the full path of the config file  
+  configDirPath = fs::path(configFilePath).parent_path();
+  if (!fs::exists(configDirPath))
+    throw std::runtime_error("Can not find the directory of the config: " + configDirPath.string());
+
   // Make sure an in and out dir were provided.
   ensureContains(json, "testDir");
   std::string testDirStr = json["testDir"];

src/main.cpp

@@ -34,7 +34,7 @@ int main(int argc, char** argv) {
 
     // Free resources
   } catch (const std::runtime_error& e) {
-    std::cout << "Test harness error: " << e.what() << '\n';
+    std::cout << e.what() << '\n';
     return 1;
   }
 

src/testharness/TestHarness.cpp

@@ -146,7 +146,7 @@ bool hasTestFiles(const fs::path& path) {
 }
 
 void TestHarness::addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file) {
-  auto testfile = std::make_unique<TestFile>(file);
+  auto testfile = std::make_unique<TestFile>(file, testArtifactsPath);
 
   TestParser parser(testfile.get());
 

src/tests/TestFile.cpp

@@ -1,6 +1,8 @@
 #include "tests/TestFile.h"
 #include "tests/TestParser.h"
 
+static uint64_t nextId = 0;
+
 namespace {
 
 std::string stripFileExtension(const std::string& str) {
@@ -12,30 +14,58 @@ std::string stripFileExtension(const std::string& str) {
 
 namespace tester {
 
-uint64_t TestFile::nextId = 0;
+// Initialize the static id to zero
+std::atomic<uint64_t> TestFile::nextId(0);
+
+// An atoimc 
+uint64_t TestFile::generateId() {
+    return nextId.fetch_add(1, std::memory_order_relaxed);
+}
+
+TestFile::TestFile(const fs::path& path, const fs::path& artifactDir)
+    : id(generateId()), testPath(path) {
 
-TestFile::TestFile(const fs::path& path) : id(++nextId), testPath(path) {
+  try {
+    // Create .test-artifacts if it doesn't exist    
+    if (!fs::exists(artifactDir)) {
+      fs::create_directories(artifactDir);
+    }
 
-  // create a unique temporary file to use as the inputs stream path
-  std::string fileInsPath = stripFileExtension(testPath.filename());
-  insPath = fs::temp_directory_path() / (fileInsPath + std::to_string(id) + ".ins");
-  outPath = fs::temp_directory_path() / (fileInsPath + std::to_string(id) + ".out");
+    // create .test-artifacts/testfiles if it doesn't exist
+    fs::path testArtifactsDir = artifactDir / "testfiles"; 
+    if (!fs::exists(testArtifactsDir)) {
+      fs::create_directories(testArtifactsDir);
+    }
 
-  std::ofstream makeInsFile(insPath);
-  std::ofstream makeOutFile(outPath);
+    std::string testName = path.stem();
+    fs::path basePath = testArtifactsDir / fs::path(testName + '-' + std::to_string(id));
 
-  // closing creates the files
-  makeInsFile.close();
-  makeOutFile.close();
+    setInsPath(fs::path(basePath.string() + ".ins"));
+    setOutPath(fs::path(basePath.string() + ".out"));
+
+    // std::cout << "Creating file: " << testName << std::endl;
+    // std::cout << "INS: " << getInsPath() << std::endl;  
+    // std::cout << "OUT: " << getOutPath() << std::endl;  
+
+  } catch (const fs::filesystem_error& e) {
+    throw std::runtime_error("Filesystem error: " + std::string(e.what()));
+  } 
 }
 
 TestFile::~TestFile() {
-  // clean up allocated resources on Testfile de-allocation
-  if (usesInputStream && !usesInputFile) {
-    fs::remove(insPath);
-  }
-  if (usesOutStream && !usesOutFile) {
-    fs::remove(outPath);
+
+  // std::cout << "Calling Destructor...\n";
+  try {
+    if (fs::exists(insPath)) {
+      // Remove temporary input stream file 
+      fs::remove(insPath);
+    }
+    if (fs::exists(outPath)) {
+      // Remove the tenmporary testfile directory and the expected out
+      fs::remove(outPath);
+    }
+  } catch (const std::exception& e) {
+    std::cerr << "Caught exception in destructor: "<< e.what() << std::endl;
   }
 }