diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index 6862c58a8..ee9e12991 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -67,7 +67,10 @@ jobs: --env LD_PRELOAD=libcapio_posix.so \ --name capio-docker \ alphaunito/capio:latest \ - capio_server_unit_tests \ + jq -n --arg pwd $(pwd) \ + '{name: "CAPIO", IO_Graph: [], exclude: [$pwd + "/aNonExistingFile", $pwd, $pwd + "/"]}' \ + > test_config.json && \ + capio_syscall_unit_tests \ --gtest_break_on_failure \ --gtest_print_time=1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bbd164c4..dc0842141 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,16 @@ IF (CAPIO_LOG AND CMAKE_BUILD_TYPE STREQUAL "Debug") include_directories("${PROJECT_BINARY_DIR}/include/syscall") ENDIF (CAPIO_LOG AND CMAKE_BUILD_TYPE STREQUAL "Debug") + +##################################### +# Global required dependencies +##################################### +FetchContent_Declare( + capio_cl + GIT_REPOSITORY https://github.com/High-Performance-IO/CAPIO-CL.git + GIT_TAG v1.3.4 +) + ##################################### # Targets ##################################### diff --git a/capio/server/CMakeLists.txt b/capio/server/CMakeLists.txt index a7f5cf2db..7d6788fa5 100644 --- a/capio/server/CMakeLists.txt +++ b/capio/server/CMakeLists.txt @@ -19,12 +19,6 @@ FetchContent_Declare( set(ARGS_BUILD_EXAMPLE OFF CACHE INTERNAL "") set(ARGS_BUILD_UNITTESTS OFF CACHE INTERNAL "") -FetchContent_Declare( - capio_cl - GIT_REPOSITORY https://github.com/High-Performance-IO/CAPIO-CL.git - GIT_TAG v1.3.4 -) - FetchContent_MakeAvailable(args capio_cl) ##################################### diff --git a/capio/server/include/client-manager/client_manager.hpp b/capio/server/include/client-manager/client_manager.hpp index 8c963b55b..3c91eb705 100644 --- a/capio/server/include/client-manager/client_manager.hpp +++ b/capio/server/include/client-manager/client_manager.hpp @@ -17,12 +17,20 @@ class ClientManager { CircularBuffer requests; std::unordered_map> responses; + /// @brief default app name + const std::string default_app_name = CAPIO_DEFAULT_APP_NAME; + /** * Data buffers variables */ struct ClientDataBuffers { - SPSCQueue *ClientToServer; - SPSCQueue *ServerToClient; + SPSCQueue ClientToServer; + SPSCQueue ServerToClient; + + /// @brief Constructor for struct so that try_emplace can be called with no explicit call to + /// new() + ClientDataBuffers(const std::string &clientToServerName, + const std::string &serverToClientName, const std::string &workflow_name); }; std::unordered_map data_buffers; diff --git a/capio/server/src/client_manager.cpp b/capio/server/src/client_manager.cpp index df69215a9..aef6d9538 100644 --- a/capio/server/src/client_manager.cpp +++ b/capio/server/src/client_manager.cpp @@ -8,6 +8,12 @@ #include "utils/capiocl_adapter.hpp" #include "utils/common.hpp" +ClientManager::ClientDataBuffers::ClientDataBuffers(const std::string &clientToServerName, + const std::string &serverToClientName, + const std::string &wf_name) + : ClientToServer(clientToServerName, get_cache_lines(), get_cache_line_size(), wf_name), + ServerToClient(serverToClientName, get_cache_lines(), get_cache_line_size(), wf_name) {} + ClientManager::ClientManager() : requests{SHM_COMM_CHAN_NAME, CAPIO_REQ_BUFF_CNT, CAPIO_REQ_MAX_SIZE, CapioCLEngine::get().getWorkflowName()} { @@ -21,13 +27,9 @@ ClientManager::~ClientManager() { void ClientManager::registerClient(pid_t tid, const std::string &app_name, const bool wait) { START_LOG(gettid(), "call(tid=%ld, app_name=%s)", tid, app_name.c_str()); - ClientDataBuffers buffers{ - new SPSCQueue(SHM_SPSC_PREFIX_WRITE + std::to_string(tid), get_cache_lines(), - get_cache_line_size(), CapioCLEngine::get().getWorkflowName()), - new SPSCQueue(SHM_SPSC_PREFIX_READ + std::to_string(tid), get_cache_lines(), - get_cache_line_size(), CapioCLEngine::get().getWorkflowName())}; - - data_buffers.emplace(tid, buffers); + data_buffers.try_emplace(tid, SHM_SPSC_PREFIX_WRITE + std::to_string(tid), + SHM_SPSC_PREFIX_READ + std::to_string(tid), + CapioCLEngine::get().getWorkflowName()); app_names.emplace(tid, app_name); files_created_by_producer.emplace(tid, std::initializer_list{}); files_created_by_app_name.emplace(app_name, std::initializer_list{}); @@ -63,8 +65,6 @@ void ClientManager::unlockClonedChild(const pid_t tid) { void ClientManager::removeClient(const pid_t tid) { START_LOG(gettid(), "call(tid=%ld)", tid); if (const auto it_resp = data_buffers.find(tid); it_resp != data_buffers.end()) { - delete it_resp->second.ClientToServer; - delete it_resp->second.ServerToClient; data_buffers.erase(it_resp); } const std::string &app_name = this->getAppName(tid); @@ -87,10 +87,10 @@ void ClientManager::replyToClient(const int tid, const off64_t offset, char *buf if (const auto out = data_buffers.find(tid); out != data_buffers.end()) { this->replyToClient(tid, offset + count); - out->second.ServerToClient->write(buf + offset, count); + out->second.ServerToClient.write(buf + offset, count); return; } - LOG("Err: no such buffer for provided tid"); + throw std::runtime_error("Err: no such buffer for provided tid"); } // NOTE: do not use const reference for path here as the emplace method leaves the original in an @@ -153,15 +153,15 @@ const std::vector &ClientManager::getProducedFiles(const pid_t tid) const std::string &ClientManager::getAppName(const pid_t tid) const { START_LOG(gettid(), "call(tid=%ld)", tid); - static std::string default_app_name = CAPIO_DEFAULT_APP_NAME; if (const auto itm = app_names.find(tid); itm != app_names.end()) { return itm->second; + } else { + return default_app_name; } - return default_app_name; } SPSCQueue &ClientManager::getClientToServerDataBuffers(const pid_t tid) { - return *data_buffers.at(tid).ClientToServer; + return data_buffers.at(tid).ClientToServer; } size_t ClientManager::getConnectedPosixClients() const { return data_buffers.size(); } diff --git a/capio/server/src/storage_manager.cpp b/capio/server/src/storage_manager.cpp index d4f7eb55a..4981d4deb 100644 --- a/capio/server/src/storage_manager.cpp +++ b/capio/server/src/storage_manager.cpp @@ -187,7 +187,7 @@ CapioFile &StorageManager::add(const std::filesystem::path &path, bool is_dir, s auto n_close_count = CapioCLEngine::get().getCommitCloseCount(path); if (n_file > 1) { - // NODE: This is probably because it needs to be filled even when dealing with directories + // NOTE: This is probably because it needs to be filled even when dealing with directories init_size = CAPIO_DEFAULT_DIR_INITIAL_SIZE; } @@ -228,7 +228,7 @@ void StorageManager::clone(const pid_t parent_tid, const pid_t child_tid) { std::vector StorageManager::getPaths() const { const shared_lock_guard slg(_mutex_storage); - std::vector paths(_storage.size()); + std::vector paths; for (const auto &[file_path, _] : _storage) { paths.emplace_back(file_path); } diff --git a/capio/tests/unit/server/CMakeLists.txt b/capio/tests/unit/server/CMakeLists.txt index d872b453c..038903d49 100644 --- a/capio/tests/unit/server/CMakeLists.txt +++ b/capio/tests/unit/server/CMakeLists.txt @@ -2,9 +2,18 @@ # Target information ##################################### set(TARGET_NAME capio_server_unit_tests) + +FetchContent_MakeAvailable(capio_cl) + set(TARGET_INCLUDE_FOLDER "${PROJECT_SOURCE_DIR}/capio/server") + +file(GLOB_RECURSE SERVER_SOURCES + "${CMAKE_SOURCE_DIR}/capio/server/**/*.cpp" +) + set(TARGET_SOURCES - src/capio_file.cpp + src/main.cpp + ${SERVER_SOURCES} ) ##################################### @@ -20,12 +29,15 @@ target_sources(${TARGET_NAME} PRIVATE "${CAPIO_COMMON_HEADERS}" "${CAPIO_SERVER_HEADERS}" ) -target_include_directories(${TARGET_NAME} PRIVATE "${TARGET_INCLUDE_FOLDER}/include") +target_include_directories(${TARGET_NAME} PRIVATE + "${TARGET_INCLUDE_FOLDER}/include" + ${capio_cl_SOURCE_DIR} +) ##################################### # Link libraries ##################################### -target_link_libraries(${TARGET_NAME} PRIVATE GTest::gtest_main rt) +target_link_libraries(${TARGET_NAME} PRIVATE GTest::gtest_main rt libcapio_cl) ##################################### # Configure tests @@ -34,6 +46,22 @@ gtest_discover_tests(${TARGET_NAME} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ) +##################################### +# Code coverage +##################################### +IF (ENABLE_COVERAGE) + IF (CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_definitions(${TARGET_NAME} PRIVATE CAPIO_COVERAGE) + target_compile_options(${TARGET_NAME} PRIVATE --coverage -fprofile-arcs -ftest-coverage) + target_link_options(${TARGET_NAME} PRIVATE --coverage -fprofile-arcs -ftest-coverage) + IF (CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") + target_link_libraries(${TARGET_NAME} PRIVATE gcov) + ENDIF (CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") + ELSE (CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "Code coverage is disabled in release mode.") + ENDIF (CMAKE_BUILD_TYPE STREQUAL "Debug") +ENDIF (ENABLE_COVERAGE) + ##################################### # Install rules ##################################### diff --git a/capio/tests/unit/server/src/capio_file.cpp b/capio/tests/unit/server/src/capio_file.hpp similarity index 95% rename from capio/tests/unit/server/src/capio_file.cpp rename to capio/tests/unit/server/src/capio_file.hpp index 88ab96d90..fab618e65 100644 --- a/capio/tests/unit/server/src/capio_file.cpp +++ b/capio/tests/unit/server/src/capio_file.hpp @@ -1,7 +1,5 @@ -#include - -#include - +#ifndef CAPIO_CAPIO_FILE_HPP +#define CAPIO_CAPIO_FILE_HPP #include "common/env.hpp" #include "utils/capio_file.hpp" @@ -59,4 +57,5 @@ TEST(ServerTest, TestInsertTwoOverlappingSectorsNested) { auto §ors = c_file.get_sectors(); EXPECT_EQ(sectors.size(), 1); EXPECT_NE(sectors.find({1L, 4L}), sectors.end()); -} \ No newline at end of file +} +#endif // CAPIO_CAPIO_FILE_HPP diff --git a/capio/tests/unit/server/src/client_manager.hpp b/capio/tests/unit/server/src/client_manager.hpp new file mode 100644 index 000000000..1f5a2fa5c --- /dev/null +++ b/capio/tests/unit/server/src/client_manager.hpp @@ -0,0 +1,57 @@ +#ifndef CAPIO_CLIENT_MANAGER_HPP +#define CAPIO_CLIENT_MANAGER_HPP + +TEST(ClientManagerTestEnvironment, testReplyToNonClient) { + char buffer[1024]; + EXPECT_THROW(client_manager->replyToClient(-1, 0, buffer, 0), std::runtime_error); +} + +TEST(ClientManagerTestEnvironment, testGetNumberOfConnectedClients) { + + EXPECT_EQ(client_manager->getConnectedPosixClients(), 0); + + client_manager->registerClient(1234); + EXPECT_EQ(client_manager->getConnectedPosixClients(), 1); + + client_manager->removeClient(1234); + EXPECT_EQ(client_manager->getConnectedPosixClients(), 0); +} + +TEST(ClientManagerTestEnvironment, testFailedRequestCode) { + + // NOTE: there is no need to delete this object as it is only attaching to the shm allocated by + // client_manager. Also calling delete on this raises std::terminate as an exception is thrown + // in the destructor + // TODO: change behaviour of ERR_EXIT to not throw exceptions but only print error and continue + const auto request_queue = + new CircularBuffer(SHM_COMM_CHAN_NAME, CAPIO_REQ_BUFF_CNT, CAPIO_REQ_MAX_SIZE); + + char req[CAPIO_REQ_MAX_SIZE], new_req[CAPIO_REQ_MAX_SIZE]; + constexpr int TEST_REQ_CODE = 123; + sprintf(req, "%04d aaaa bbbb cccc", TEST_REQ_CODE); + request_queue->write(req, CAPIO_REQ_MAX_SIZE); + const auto return_code = client_manager->readNextRequest(new_req); + std::cout << new_req << std::endl; + EXPECT_EQ(return_code, TEST_REQ_CODE); + + sprintf(req, "abc aaaa bbbb cccc"); + request_queue->write(req, CAPIO_REQ_MAX_SIZE); + + EXPECT_EQ(client_manager->readNextRequest(new_req), -1); +} + +TEST(ClientManagerTestEnvironment, testAddAndRemoveProducedFiles) { + + client_manager->registerClient(1234, "test_app"); + client_manager->registerProducedFile(1234, "test.txt"); + + EXPECT_TRUE(client_manager->isProducer(1234, "test.txt")); + + client_manager->removeProducedFile(1234, "test.txt"); + EXPECT_FALSE(client_manager->isProducer(1234, "test.txt")); + + client_manager->registerProducedFile(1111, "test1.txt"); + EXPECT_FALSE(client_manager->isProducer(1111, "test1.txt")); +} + +#endif // CAPIO_CLIENT_MANAGER_HPP diff --git a/capio/tests/unit/server/src/main.cpp b/capio/tests/unit/server/src/main.cpp new file mode 100644 index 000000000..b9e73ec58 --- /dev/null +++ b/capio/tests/unit/server/src/main.cpp @@ -0,0 +1,51 @@ +#include + +char *node_name; + +#include "capiocl.hpp" +#include "capiocl/engine.h" +#include "client-manager/client_manager.hpp" +#include "common/constants.hpp" +#include "storage/manager.hpp" +#include "utils/capiocl_adapter.hpp" +#include "utils/location.hpp" + +capiocl::engine::Engine *capio_cl_engine; +StorageManager *storage_manager = nullptr; +ClientManager *client_manager = nullptr; + +const capiocl::engine::Engine &CapioCLEngine::get() { return *capio_cl_engine; } + +class ServerUnitTestEnvironment : public testing::Environment { + public: + explicit ServerUnitTestEnvironment() = default; + + void SetUp() override { + capio_cl_engine = new capiocl::engine::Engine(false); + node_name = new char[HOST_NAME_MAX]; + gethostname(node_name, HOST_NAME_MAX); + open_files_location(); + + client_manager = new ClientManager(); + storage_manager = new StorageManager(); + } + + void TearDown() override { + delete storage_manager; + delete client_manager; + delete capio_cl_engine; + } +}; + +/// Include test sources + +#include "capio_file.hpp" +#include "client_manager.hpp" +#include "storage_manager.hpp" + +int main(int argc, char **argv, char **envp) { + testing::InitGoogleTest(&argc, argv); + + testing::AddGlobalTestEnvironment(new ServerUnitTestEnvironment()); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/capio/tests/unit/server/src/storage_manager.hpp b/capio/tests/unit/server/src/storage_manager.hpp new file mode 100644 index 000000000..266d64b23 --- /dev/null +++ b/capio/tests/unit/server/src/storage_manager.hpp @@ -0,0 +1,109 @@ +#ifndef CAPIO_STORAGE_MANAGER_TEST_HPP +#define CAPIO_STORAGE_MANAGER_TEST_HPP + +TEST(StorageManagerTestEnvironment, testGetPaths) { + + std::vector test_file_paths = { + "test1.txt", + "test2.txt", + "test3.txt", + }; + + for (const auto &path : test_file_paths) { + storage_manager->add(path, false, 0); + } + + const auto storage_paths = storage_manager->getPaths(); + + EXPECT_EQ(storage_paths.size(), test_file_paths.size()); + + for (const auto &path : test_file_paths) { + EXPECT_TRUE(std::find(storage_paths.begin(), storage_paths.end(), path) != + storage_paths.end()); + } +} + +TEST(StorageManagerTestEnvironment, testExceptions) { + + EXPECT_THROW(storage_manager->get("test.txt"), std::runtime_error); + EXPECT_THROW(storage_manager->get(1234, 1234), std::runtime_error); +} + +TEST(StorageManagerTestEnvironment, testInitDirectory) { + + capio_cl_engine->setDirectory("myDirectory"); + capio_cl_engine->setDirectoryFileCount("myDirectory", 10); + + storage_manager->add("myDirectory", true, 0); + + const auto &dir = storage_manager->get("myDirectory"); + + EXPECT_EQ(dir.get_buf_size(), CAPIO_DEFAULT_DIR_INITIAL_SIZE); + + storage_manager->updateDirectory(1, "myDirectory"); + const auto &dir1 = storage_manager->get("myDirectory"); + + EXPECT_FALSE(dir1.first_write); +} + +TEST(StorageManagerTestEnvironment, testAddDirectoryFailure) { + char *old_capio_dir = getenv("CAPIO_DIR"); + setenv("CAPIO_DIR", "/", 1); + open_files_location(); + + storage_manager->add("/tmp", true, 0); + EXPECT_EQ(storage_manager->addDirectory(1, "/tmp/newDirectoryFail"), 0); + EXPECT_EQ(storage_manager->addDirectory(1, "/tmp/newDirectoryFail"), 1); + if (old_capio_dir != nullptr) { + setenv("CAPIO_DIR", old_capio_dir, 1); + } +} + +TEST(StorageManagerTestEnvironment, testRemameFile) { + + storage_manager->add("oldName", false, 0); + storage_manager->add("oldNameNoChange", false, 0); + + storage_manager->addFileToTid(1234, 3, "oldName", 0); + storage_manager->addFileToTid(1234, 4, "oldNameNoChange", 0); + + storage_manager->add("oldNameNoChange", false, 0); + storage_manager->rename("oldName", "newName"); + + EXPECT_THROW(storage_manager->get("oldName"), std::runtime_error); + EXPECT_NO_THROW(storage_manager->get("oldNameNoChange")); +} + +TEST(StorageManagerTestEnvironment, testNumberOfOpensAndCloses) { + + storage_manager->add("myFile", false, 0); + storage_manager->addFileToTid(1234, 3, "myFile", 0); + storage_manager->addFileToTid(1234, 4, "myFile", 0); + + EXPECT_FALSE(storage_manager->get("myFile").is_deletable()); + + storage_manager->get("myFile").close(); + EXPECT_FALSE(storage_manager->get("myFile").is_deletable()); + + storage_manager->get("myFile").close(); + EXPECT_TRUE(storage_manager->get("myFile").is_deletable()); +} + +TEST(StorageManagerTestEnvironment, testNumberOfOpensAfterClone) { + + storage_manager->add("myFile", false, 0); + storage_manager->addFileToTid(1234, 3, "myFile", 0); + storage_manager->clone(1234, 5678); + + EXPECT_FALSE(storage_manager->get("myFile").is_deletable()); + + storage_manager->removeFromTid(1234, 3); + EXPECT_FALSE(storage_manager->get("myFile").is_deletable()); + + storage_manager->removeFromTid(5678, 3); + storage_manager->get("myFile").close(); + + EXPECT_TRUE(storage_manager->get("myFile").is_deletable()); +} + +#endif // CAPIO_STORAGE_MANAGER_TEST_HPP