diff --git a/include/config/Config.h b/include/config/Config.h index 2979d767..90c32e50 100644 --- a/include/config/Config.h +++ b/include/config/Config.h @@ -28,6 +28,9 @@ class Config { const std::optional& getDebugPath() const { return debugPackage; } const std::optional& getFailureLogPath() const { return failureLogPath; }; + // Return the path which the JSON config lives in + const fs::path&getConfigDirPath() const { return configDirPath; } + // Non optional config variables const fs::path&getTestDirPath() const { return testDirPath; } @@ -64,7 +67,7 @@ class Config { std::optional failureLogPath; std::optional debugPackage; - fs::path testDirPath; + fs::path testDirPath, configDirPath; // Option file maps. PathMap executables; diff --git a/include/testharness/TestHarness.h b/include/testharness/TestHarness.h index ac32e329..40529b6c 100644 --- a/include/testharness/TestHarness.h +++ b/include/testharness/TestHarness.h @@ -28,7 +28,17 @@ class TestHarness { TestHarness() = delete; // Construct the Tester with a parsed JSON file. - TestHarness(const Config& cfg) : cfg(cfg), results() { findTests(); } + TestHarness(const Config& cfg) + : cfg(cfg), + results() + { + // Create temporary dir for test and toolchain files + testArtifactsPath = fs::path(cfg.getConfigDirPath() / ".test-artifacts"); + fs::create_directory(testArtifactsPath); + + // Find tests + findTests(); + } // Returns true if any tests failed, false otherwise. bool runTests(); @@ -52,6 +62,9 @@ class TestHarness { // let derived classes find tests. void findTests(); + // Create a local tmp path for ephemeral test input, output and toolchain files + fs::path testArtifactsPath; + private: // The results of the tests. ResultManager results; diff --git a/include/tests/TestFile.h b/include/tests/TestFile.h index 817baf84..4c1723e1 100644 --- a/include/tests/TestFile.h +++ b/include/tests/TestFile.h @@ -7,6 +7,8 @@ #include #include #include +#include +#include namespace fs = std::filesystem; @@ -25,42 +27,46 @@ class TestParser; class TestFile { public: TestFile() = delete; - // construct Testfile from path to .test file. - TestFile(const fs::path& path); + TestFile(const fs::path& path, const fs::path& artifactDir); ~TestFile(); - uint64_t id; + uint64_t getId() const { return id; } - // getters - fs::path getTestPath() const { return testPath; } - fs::path getInsPath() const { return insPath; } - fs::path getOutPath() const { return outPath; } + // Test path getters + const fs::path& getTestPath() const { return testPath; } + const fs::path& getOutPath() const { return outPath; } + const fs::path& getInsPath() const { return insPath; } + + // Test state getters ParseError getParseError() const { return errorState; } std::string getParseErrorMsg() const; double getElapsedTime() const { return elapsedTime; } bool didError() const { return errorState != ParseError::NoError; } - // setters - void setTestPath(fs::path path) { testPath = path; } + // Test path getters void setInsPath(fs::path path) { insPath = path; } void setOutPath(fs::path path) { outPath = path; } - void getParseError(ParseError error) { errorState = error; } + + // Test path setters + void setParseError(ParseError error) { errorState = error; } void setParseErrorMsg(std::string msg) { errorMsg = msg; } void setElapsedTime(double elapsed) { elapsedTime = elapsed; } - // if test has any input and if test uses input file specifically - bool usesInputStream{false}, usesInputFile{false}; - bool usesOutStream{false}, usesOutFile{false}; - friend class TestParser; +private: + static uint64_t generateId(); + protected: - static uint64_t nextId; + static std::atomic nextId; private: + uint64_t id; // Path for the test, ins and out files - fs::path testPath, insPath, outPath; + fs::path testPath; + fs::path outPath; + fs::path insPath; // Test file breaks some convention or was unable to parse directives. ParseError errorState{ParseError::NoError}; diff --git a/include/tests/TestParser.h b/include/tests/TestParser.h index a72973d3..6614042a 100644 --- a/include/tests/TestParser.h +++ b/include/tests/TestParser.h @@ -5,6 +5,7 @@ #include "TestFile.h" #include #include +#include namespace fs = std::filesystem; @@ -42,8 +43,7 @@ class TestParser { // helper method to insert a newline prefixed line to a file void insLineToFile(fs::path filePath, std::string line, bool firstInsert); - // methods below look for INPUT, CHECK, INPUT_FILE, CHECK_FILE directive in - // a lines + // identifiy for INPUT, CHECK, INPUT_FILE or CHECK_FILE directive in a line ParseError matchInputDirective(std::string& line); ParseError matchCheckDirective(std::string& line); ParseError matchInputFileDirective(std::string& line); diff --git a/include/toolchain/Command.h b/include/toolchain/Command.h index 4ddfe392..e6b3e20e 100644 --- a/include/toolchain/Command.h +++ b/include/toolchain/Command.h @@ -32,7 +32,7 @@ class Command { Command(const Command& command) = default; // Destructor for removing temporary files - ~Command() {} + ~Command(); // Execute the command. ExecutionOutput execute(const ExecutionInput& ei) const; @@ -58,15 +58,13 @@ class Command { std::string name; fs::path exePath; fs::path runtimePath; + fs::path tmpPath; std::vector args; - + // Every command produces a file descriptor for each of these paths fs::path errPath; fs::path outPath; - // The command can supply a output file to use instead of stdout/err - std::optional outputFile; - // Uses runtime and uses input stream. bool usesRuntime, usesInStr; diff --git a/include/toolchain/ExecutionState.h b/include/toolchain/ExecutionState.h index 1de14f46..588b3a56 100644 --- a/include/toolchain/ExecutionState.h +++ b/include/toolchain/ExecutionState.h @@ -1,6 +1,7 @@ #ifndef TESTER_EXECUTION_STATE_H #define TESTER_EXECUTION_STATE_H +#include #include #include namespace fs = std::filesystem; @@ -16,8 +17,14 @@ class ExecutionInput { // Creates input to a subprocess execution. ExecutionInput(fs::path inputPath, fs::path inputStreamPath, fs::path testedExecutable, fs::path testedRuntime) - : inputPath(std::move(inputPath)), inputStreamPath(std::move(inputStreamPath)), - testedExecutable(std::move(testedExecutable)), testedRuntime(std::move(testedRuntime)) {} + : inputPath(std::move(inputPath)), + inputStreamPath(std::move(inputStreamPath)), + testedExecutable(std::move(testedExecutable)), + testedRuntime(std::move(testedRuntime)) {} + + ~ExecutionInput() { + // std::cout << "Destory execution input: " << inputPath << std::endl; + } // Gets input file. const fs::path& getInputFile() const { return inputPath; } @@ -50,6 +57,10 @@ class ExecutionOutput { errPath(std::move(errPath)), rv(0), elapsedTime(0), hasElapsed(false), isErrorTest(false) {} + ~ExecutionOutput() { + // std::cout << "Destory execution output: " << outPath << std::endl; + } + // Gets output file. fs::path getOutputFile() const { return outPath; } fs::path getErrorFile() const { return errPath; } diff --git a/src/config/Config.cpp b/src/config/Config.cpp index 15221ad0..621533fe 100644 --- a/src/config/Config.cpp +++ b/src/config/Config.cpp @@ -55,6 +55,11 @@ Config::Config(int argc, char** argv) : timeout(2l) { JSON json; jsonFile >> json; + // Set the config path from the full path of the config file + configDirPath = fs::path(configFilePath).parent_path(); + if (!fs::exists(configDirPath)) + throw std::runtime_error("Can not find the directory of the config: " + configDirPath.string()); + // Make sure an in and out dir were provided. ensureContains(json, "testDir"); std::string testDirStr = json["testDir"]; diff --git a/src/main.cpp b/src/main.cpp index 9f940cef..819cb736 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) { // Free resources } catch (const std::runtime_error& e) { - std::cout << "Test harness error: " << e.what() << '\n'; + std::cout << e.what() << '\n'; return 1; } diff --git a/src/testharness/TestHarness.cpp b/src/testharness/TestHarness.cpp index a6bc59c5..694dd04d 100644 --- a/src/testharness/TestHarness.cpp +++ b/src/testharness/TestHarness.cpp @@ -146,7 +146,7 @@ bool hasTestFiles(const fs::path& path) { } void TestHarness::addTestFileToSubPackage(SubPackage& subPackage, const fs::path& file) { - auto testfile = std::make_unique(file); + auto testfile = std::make_unique(file, testArtifactsPath); TestParser parser(testfile.get()); diff --git a/src/tests/TestFile.cpp b/src/tests/TestFile.cpp index c91d180b..35eb714e 100644 --- a/src/tests/TestFile.cpp +++ b/src/tests/TestFile.cpp @@ -1,6 +1,8 @@ #include "tests/TestFile.h" #include "tests/TestParser.h" +static uint64_t nextId = 0; + namespace { std::string stripFileExtension(const std::string& str) { @@ -12,30 +14,58 @@ std::string stripFileExtension(const std::string& str) { namespace tester { -uint64_t TestFile::nextId = 0; +// Initialize the static id to zero +std::atomic TestFile::nextId(0); + +// An atoimc +uint64_t TestFile::generateId() { + return nextId.fetch_add(1, std::memory_order_relaxed); +} + +TestFile::TestFile(const fs::path& path, const fs::path& artifactDir) + : id(generateId()), testPath(path) { -TestFile::TestFile(const fs::path& path) : id(++nextId), testPath(path) { + try { + // Create .test-artifacts if it doesn't exist + if (!fs::exists(artifactDir)) { + fs::create_directories(artifactDir); + } - // create a unique temporary file to use as the inputs stream path - std::string fileInsPath = stripFileExtension(testPath.filename()); - insPath = fs::temp_directory_path() / (fileInsPath + std::to_string(id) + ".ins"); - outPath = fs::temp_directory_path() / (fileInsPath + std::to_string(id) + ".out"); + // create .test-artifacts/testfiles if it doesn't exist + fs::path testArtifactsDir = artifactDir / "testfiles"; + if (!fs::exists(testArtifactsDir)) { + fs::create_directories(testArtifactsDir); + } - std::ofstream makeInsFile(insPath); - std::ofstream makeOutFile(outPath); + std::string testName = path.stem(); + fs::path basePath = testArtifactsDir / fs::path(testName + '-' + std::to_string(id)); - // closing creates the files - makeInsFile.close(); - makeOutFile.close(); + setInsPath(fs::path(basePath.string() + ".ins")); + setOutPath(fs::path(basePath.string() + ".out")); + + // std::cout << "Creating file: " << testName << std::endl; + // std::cout << "INS: " << getInsPath() << std::endl; + // std::cout << "OUT: " << getOutPath() << std::endl; + + } catch (const fs::filesystem_error& e) { + throw std::runtime_error("Filesystem error: " + std::string(e.what())); + } } TestFile::~TestFile() { - // clean up allocated resources on Testfile de-allocation - if (usesInputStream && !usesInputFile) { - fs::remove(insPath); - } - if (usesOutStream && !usesOutFile) { - fs::remove(outPath); + + // std::cout << "Calling Destructor...\n"; + try { + if (fs::exists(insPath)) { + // Remove temporary input stream file + fs::remove(insPath); + } + if (fs::exists(outPath)) { + // Remove the tenmporary testfile directory and the expected out + fs::remove(outPath); + } + } catch (const std::exception& e) { + std::cerr << "Caught exception in destructor: "<< e.what() << std::endl; } } diff --git a/src/tests/TestParser.cpp b/src/tests/TestParser.cpp index d95e2df6..d5f8c8fa 100644 --- a/src/tests/TestParser.cpp +++ b/src/tests/TestParser.cpp @@ -14,14 +14,35 @@ bool fullyContains(const std::string& str, const std::string& substr) { return str.substr(pos, substr.length()) == substr; } -void TestParser::insLineToFile(fs::path filePath, std::string line, bool firstInsert) { - // open in append mode since otherwise multi-line checks and inputs would - // over-write themselves. - std::ofstream out(filePath, std::ios::app); - if (!firstInsert) { - out << "\n"; +ParseError copyFile(const fs::path& from, const fs::path& to) { + + // Open the files to operate upon + std::ifstream sourceFile(from, std::ios::binary); + std::ofstream destFile(to, std::ios::binary); + + // Check for errors opening + if (!destFile || !sourceFile) { + return ParseError::FileError; } - out << line; + + // Write the contents and check for errors + destFile << sourceFile.rdbuf(); + if (sourceFile.fail() || destFile.fail()) { + return ParseError::FileError; + } + + return ParseError::NoError; +} + +void TestParser::insLineToFile(fs::path filePath, std::string line, bool firstInsert) { + + if (firstInsert) { + std::ofstream out(filePath); + out << line; + } else { + std::ofstream out(filePath, std::ios::app); + out << "\n" << line; + } } /** @@ -62,13 +83,9 @@ ParseError TestParser::matchInputDirective(std::string& line) { size_t findIdx = line.find(Directive::INPUT); std::string inputLine = line.substr(findIdx + Directive::INPUT.length()); - try { - insLineToFile(testfile->getInsPath(), inputLine, !foundInput); - } catch (...) { - return ParseError::FileError; - } - + insLineToFile(testfile->getInsPath(), inputLine, !foundInput); foundInput = true; + return ParseError::NoError; } @@ -82,17 +99,13 @@ ParseError TestParser::matchCheckDirective(std::string& line) { return ParseError::NoError; if (foundCheckFile) return ParseError::DirectiveConflict; - + size_t findIdx = line.find(Directive::CHECK); std::string checkLine = line.substr(findIdx + Directive::CHECK.length()); - try { - insLineToFile(testfile->getOutPath(), checkLine, !foundCheck); - } catch (...) { - return ParseError::FileError; - } - + insLineToFile(testfile->getOutPath(), checkLine, !foundCheck); foundCheck = true; + return ParseError::NoError; } @@ -107,13 +120,16 @@ ParseError TestParser::matchInputFileDirective(std::string& line) { if (foundInput) return ParseError::DirectiveConflict; - PathOrError pathOrError = parsePathFromLine(line, Directive::INPUT_FILE); - if (std::holds_alternative(pathOrError)) { - testfile->setInsPath(std::get(pathOrError)); + PathOrError path = parsePathFromLine(line, Directive::INPUT_FILE); + if (std::holds_alternative(path)) { + // Copy the input file referenced into the testfiles ephemeral ins file + auto inputPath = std::get(path); + copyFile(inputPath, testfile->getInsPath()); foundInputFile = true; return ParseError::NoError; } - return std::get(pathOrError); + + return std::get(path); } /** @@ -127,14 +143,16 @@ ParseError TestParser::matchCheckFileDirective(std::string& line) { if (foundCheck) return ParseError::DirectiveConflict; - PathOrError pathOrError = parsePathFromLine(line, Directive::CHECK_FILE); - if (std::holds_alternative(pathOrError)) { - testfile->setOutPath(std::get(pathOrError)); - foundCheckFile = true; + PathOrError path = parsePathFromLine(line, Directive::CHECK_FILE); + if (std::holds_alternative(path)) { + // Copy the input file referenced into the testfiles ephemeral ins file + auto outputPath = std::get(path); + copyFile(outputPath, testfile->getOutPath()); + foundCheckFile= true; return ParseError::NoError; } - return std::get(pathOrError); + return std::get(path); } /** @@ -163,7 +181,7 @@ ParseError TestParser::matchDirectives(std::string& line) { * contained with in a comment. Use comment state in class instance to track. */ void TestParser::trackCommentState(std::string& line) { -std::string result; + std::string result; inLineComment = false; // reset line comment for (unsigned int i = 0; i < line.length(); i++) { @@ -217,19 +235,13 @@ void TestParser::parse() { if (!line.empty()) { ParseError error = matchDirectives(line); if (error != ParseError::NoError) { - testfile->getParseError(error); + testfile->setParseError(error); testfile->setParseErrorMsg("Generic Error"); break; } } } - // Set final flags to update test state - testfile->usesInputStream = (foundInput || foundInputFile); - testfile->usesInputFile = (foundInputFile); - testfile->usesOutStream = (foundCheck || foundCheckFile); - testfile->usesOutFile = foundCheckFile; - testFileStream.close(); } diff --git a/src/tests/TestRunning.cpp b/src/tests/TestRunning.cpp index a4ac1015..2699f74f 100644 --- a/src/tests/TestRunning.cpp +++ b/src/tests/TestRunning.cpp @@ -149,10 +149,21 @@ void formatFileDump(const fs::path& testPath, const fs::path& expOutPath, const fs::path& genOutPath) { std::cout << "----- TestFile: "<< testPath.filename() << std::endl; dumpFile(testPath); - std::cout << "----- Expected Output (" << fs::file_size(expOutPath) << " bytes)" << std::endl; - dumpFile(expOutPath, true); - std::cout << "----- Generated Output (" << fs::file_size(genOutPath) << " bytes)" << std::endl; - dumpFile(genOutPath, true); + + if (fs::exists(expOutPath)) { + std::cout << "----- Expected Output (" << fs::file_size(expOutPath) << " bytes)" << std::endl; + dumpFile(expOutPath, true); + } else { + std::cout << "----- Expected Output: (0 bytes)" << std::endl; + dumpFile("/dev/null", true); + } + if (fs::exists(genOutPath)) { + std::cout << "----- Generated Output (" << fs::file_size(genOutPath) << " bytes)" << std::endl; + dumpFile(genOutPath, true); + } else { + std::cout << "----- Generated Output: (0 bytes)" << std::endl; + dumpFile("/dev/null", true); + } std::cout << "-----------------------" << std::endl; } @@ -193,7 +204,6 @@ TestResult runTest(TestFile* test, const ToolChain& toolChain, const Config& cfg const fs::path testPath = test->getTestPath(); const fs::path expOutPath = test->getOutPath(); - const fs::path insPath = test->getInsPath(); fs::path genOutPath; std::string genErrorString, expErrorString, diffString; diff --git a/src/toolchain/Command.cpp b/src/toolchain/Command.cpp index 7e12686c..01cbf60d 100644 --- a/src/toolchain/Command.cpp +++ b/src/toolchain/Command.cpp @@ -22,7 +22,7 @@ namespace { /// @brief Open the file with provided flags and mode. Redirect the file descriptor /// supplied by dup_fd to the file underlying file_str. int redirectStdStream(const std::string& file_str, int flags, mode_t mode, int dup_fd) { - + // Open the process int fd = open(file_str.c_str(), flags, mode); if (fd == -1) { @@ -78,13 +78,19 @@ void becomeCommand(const std::string& exe, // Open the supplied files and redirect FD of the current child process to them. int outFileStatus = redirectStdStream(output.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, STDOUT_FILENO); - int errorFileStatus = redirectStdStream(error.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, STDERR_FILENO); - int inFileStatus = !input.empty() ? redirectStdStream(input.c_str(), O_RDONLY, 0, STDIN_FILENO) : 0; - + int errFileStatus = redirectStdStream(error.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR, STDERR_FILENO); + int insFileStatus = !input.empty() ? redirectStdStream(input.c_str(), O_RDONLY, 0, STDIN_FILENO) : 0; + // If opening any of the supplied output, input, or error files failed, raise here. - if (outFileStatus == -1 || errorFileStatus == -1 || inFileStatus == -1) { - perror("dup2"); - exit(EXIT_FAILURE); + if (outFileStatus == -1) { + perror("dup2 failed to open output file"); + exit(EXIT_FAILURE); + } else if (errFileStatus == -1) { + perror("dup2 failed to open error file"); + exit(EXIT_FAILURE); + } else if (insFileStatus == -1) { + perror("dup2 failed to open input stream file"); + exit(EXIT_FAILURE); } // Replace ourselves with the command. @@ -106,6 +112,12 @@ void runCommand(std::promise& promise, std::atomic_bool& killVar, const std::string& error, const std::string& runtime) { +#if defined(DEBUG) + std::cerr << "Running Command: " << exe << std::endl; + std::cerr << "Made out file: " << output << std::endl; + std::cerr << "Made error file: " << error << std::endl; + std::cerr << "Made input file: " << input << std::endl; +#endif pid_t childId = fork(); // We're the child process, we want to replace our process image with the @@ -174,6 +186,9 @@ namespace tester { Command::Command(const JSON& step, int64_t timeout) : usesRuntime(false), usesInStr(false), timeout(timeout) { + + fs::path testArtifactsPath = "./.test-artifacts"; + // Make sure the step has all of the values needed for construction. ensureContains(step, "stepName"); ensureContains(step, "executablePath"); @@ -184,20 +199,22 @@ Command::Command(const JSON& step, int64_t timeout) for (std::string arg : step["arguments"]) args.push_back(arg); - // If no output path is supplied by default, temporaries are created to capture stdout and stderr. - std::string output_name = std::string(step["stepName"]) + ".stdout"; - std::string error_name = std::string(step["stepName"]) + ".stderr"; - outPath = fs::temp_directory_path() / output_name; - errPath = fs::temp_directory_path() / error_name; - // Set the executable path std::string path = step["executablePath"]; exePath = fs::path(path); - // Allow override of stdout path - if (doesContain(step, "output")) - outputFile = fs::path(step["output"]); + // Allow override of stdout path with output property + if (doesContain(step, "output")) { + outPath = testArtifactsPath / fs::path(step["output"]); + } else { + std::string outFileName = std::string(step["stepName"]) + ".stdout"; + outPath = testArtifactsPath / outFileName; + } + // Always create a stderr path + std::string errFileName = std::string(step["stepName"]) + ".stderr"; + errPath = testArtifactsPath / fs::path(errFileName); + // Do we use an input stream file? if (doesContain(step, "usesInStr")) usesInStr = step["usesInStr"]; @@ -209,12 +226,19 @@ Command::Command(const JSON& step, int64_t timeout) // Do we allow errors? if (doesContain(step, "allowError")) allowError = step["allowError"]; + + // std::cout << "Created command with outpath: " << outPath << std::endl; + // std::cout << "Created command with errPath: " << errPath << std::endl; +} + +Command::~Command() { + // std::cout << "Destroying command\n"; } ExecutionOutput Command::execute(const ExecutionInput& ei) const { + // Create our output context. - fs::path out = outputFile.has_value() ? *outputFile : outPath; - ExecutionOutput eo(out, errPath); + ExecutionOutput eo(outPath, errPath); // Always remove old output files so we know if a new one was created std::error_code ec; @@ -226,10 +250,17 @@ ExecutionOutput Command::execute(const ExecutionInput& ei) const { for (const std::string& arg : args) trueArgs.emplace_back(resolveArg(ei, eo, arg).string()); +#if defined(DEBUG) + std::cout << "OUT PATH: " << outPath << std::endl; + std::cout << "ERR PATH: " << errPath << std::endl; + std::cout << "INS PATH: " << ei.getInputStreamFile() << std::endl; +#endif // Get the runtime path and standard out file, the things used in setting up // the execution of the command. std::string runtimeStr = usesRuntime ? ei.getTestedRuntime().string() : ""; - std::string inPathStr = usesInStr ? ei.getInputStreamFile().string() : ""; + std::string inPathStr = fs::exists(ei.getInputStreamFile()) && usesInStr + ? ei.getInputStreamFile().string() + : ""; std::string outPathStr = outPath.string(); std::string errPathStr = errPath.string(); diff --git a/src/toolchain/ToolChain.cpp b/src/toolchain/ToolChain.cpp index 70441512..8cb90714 100644 --- a/src/toolchain/ToolChain.cpp +++ b/src/toolchain/ToolChain.cpp @@ -34,7 +34,9 @@ ExecutionOutput ToolChain::build(TestFile* test) const { eo.setIsErrorTest(true); return eo; } - + + // The input for the next command is the output of the previous command, along with + // the same input stream, tested executable and runtime, which is shared for all commands. ei = ExecutionInput(eo.getOutputFile(), ei.getInputStreamFile(), ei.getTestedExecutable(), ei.getTestedRuntime()); }