From 7fcb5c3f880c1e127eb7958b75f382616ef49811 Mon Sep 17 00:00:00 2001 From: Mark Redeman Date: Sun, 25 Oct 2015 19:31:09 +0100 Subject: [PATCH 1/4] Use the mcbsp c++ wrapper --- src/Domains/BoxedDomain.cpp | 2 +- src/Domains/DomainInitializer.cpp | 4 ++-- src/LBM/Simulation.cpp | 2 +- src/LBM/parallel.h | 8 -------- src/LBM/parallel_bsp.h | 1 + src/main.cpp | 2 +- 6 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 src/LBM/parallel.h create mode 100644 src/LBM/parallel_bsp.h diff --git a/src/Domains/BoxedDomain.cpp b/src/Domains/BoxedDomain.cpp index 9dc55c0..cee7517 100644 --- a/src/Domains/BoxedDomain.cpp +++ b/src/Domains/BoxedDomain.cpp @@ -1,5 +1,5 @@ #include "BoxedDomain.h" -#include "../LBM/parallel.h" +#include "../LBM/parallel_bsp.h" namespace Domains { diff --git a/src/Domains/DomainInitializer.cpp b/src/Domains/DomainInitializer.cpp index 5fa3016..b5c2763 100644 --- a/src/Domains/DomainInitializer.cpp +++ b/src/Domains/DomainInitializer.cpp @@ -4,7 +4,7 @@ #include /* ceil */ -#include "../LBM/parallel.h" +#include "../LBM/parallel_bsp.h" namespace Domains { @@ -48,7 +48,7 @@ namespace Domains { for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) { size_t idx; // the hashIdx of the current messenger - MCBSP_BYTESIZE_TYPE status; + MCBSP_BYTESIZE_TYPE status = 0; bsp_get_tag(&status,&idx); if (status > 0) diff --git a/src/LBM/Simulation.cpp b/src/LBM/Simulation.cpp index 209b817..70fa7cc 100644 --- a/src/LBM/Simulation.cpp +++ b/src/LBM/Simulation.cpp @@ -4,7 +4,7 @@ #include #include -#include "../LBM/parallel.h" +#include "../LBM/parallel_bsp.h" namespace LBM { diff --git a/src/LBM/parallel.h b/src/LBM/parallel.h deleted file mode 100644 index 675ab59..0000000 --- a/src/LBM/parallel.h +++ /dev/null @@ -1,8 +0,0 @@ -extern "C" { - // on my pc - #define MCBSP_COMPATIBILITY_MODE - #include "mcbsp.h" - - // for surfsara: - // #include -} diff --git a/src/LBM/parallel_bsp.h b/src/LBM/parallel_bsp.h new file mode 100644 index 0000000..50620c5 --- /dev/null +++ b/src/LBM/parallel_bsp.h @@ -0,0 +1 @@ +#include \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 1df6ce6..f115c0d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -11,7 +11,7 @@ #include "VelocitySets/d2q9.h" #include "LBM/node.h" -#include "LBM/parallel.h" +#include "LBM/parallel_bsp.h" #include From 53ebcd3dd9faf7589d7fa33ceba06d2d2209300e Mon Sep 17 00:00:00 2001 From: Mark Redeman Date: Sat, 31 Oct 2015 15:27:39 +0100 Subject: [PATCH 2/4] Move LBM BSP specific code to LBM_BSP_Program --- src/LBM/LBM_BSP_Program.cpp | 113 ++++++++++++++++++++++++++++ src/LBM/LBM_BSP_Program.h | 40 ++++++++++ src/LBM/Parallel/MCBSP.h | 0 src/LBM/Simulation.cpp | 75 +++++++++---------- src/LBM/Simulation.h | 2 + src/main.cpp | 142 +++--------------------------------- src/main.h | 13 ---- 7 files changed, 202 insertions(+), 183 deletions(-) create mode 100644 src/LBM/LBM_BSP_Program.cpp create mode 100644 src/LBM/LBM_BSP_Program.h create mode 100644 src/LBM/Parallel/MCBSP.h diff --git a/src/LBM/LBM_BSP_Program.cpp b/src/LBM/LBM_BSP_Program.cpp new file mode 100644 index 0000000..f362be6 --- /dev/null +++ b/src/LBM/LBM_BSP_Program.cpp @@ -0,0 +1,113 @@ +#include "LBM_BSP_Program.h" + +#include "../Domains/DomainInitializer.h" +#include "../VelocitySets/d2q9.h" + +#include +#include +#include +#include + +using namespace Domains; +using std::make_unique; +using namespace mcbsp; + +size_t REPORT_PER_ITERATION = 10; + +LBM_BSP_Program::LBM_BSP_Program(size_t iterations, std::vector domainSize) +{ + d_iterations = iterations; + d_domainSize = domainSize; +} + +void LBM_BSP_Program::spmd() { + p = bsp_nprocs(); + s = bsp_pid(); + + auto sim = setup_simulation(); + run_simulation(sim); + delete sim; +} + + +BSP_program * LBM_BSP_Program::newInstance() { + return new LBM_BSP_Program(d_iterations, d_domainSize); +} + +/** + * + * @param p amount of processors being used + * @param s current processor index + */ +LBM::Simulation * LBM_BSP_Program::setup_simulation() +{ + if (s == 0) + log_simulation_data(); + + bsp_sync(); + double initialization_time = bsp_time(); + + auto set = new D2Q9; + DomainInitializer initializer(set, d_domainSize, s, p); + + auto sim = new LBM::Simulation(initializer.domain()); + + // we'll send both the local idx and the direction which we we'll change + MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t[2]); + bsp_set_tagsize(&tag_size); + + // Log initialization time and prepare computation time + bsp_sync(); + double current_time = bsp_time(); + if (s == 0) + { + std::ofstream out("logs/timings.log", std::ios::out | std::ios::app); + // Initialization time + out << "IT: " << (current_time - initialization_time) << " sec, "; + } + return sim; +} + +/** + * Run ITERATIONS step and periodically report the current state of the simulation + * @param sim + */ +void LBM_BSP_Program::run_simulation(LBM::Simulation * sim) +{ + double process_time = bsp_time(); + + for (size_t iter = 0; iter < d_iterations; ++iter) + { + sim->step(); + + if (iter % REPORT_PER_ITERATION == 0) + sim->report(); + } + + // Create a timestamp + bsp_sync(); + double current_time = bsp_time(); + if (s == 0) + { + std::ofstream out("logs/timings.log", std::ios::out | std::ios::app); + // computation time + out << "CT: " << (current_time - process_time) << " sec" << '\n'; + } +} + +void LBM_BSP_Program::log_simulation_data() +{ + std::ofstream out("logs/timings.log", std::ios::out | std::ios::app); + size_t p = bsp_nprocs(); + // Start by writing basic info to the file + out << "LBM, p: " << p << ", it: " << d_iterations << ", ds ("; + show_vector(d_domainSize, out); + out << ", "; +} + +void LBM_BSP_Program::show_vector(std::vector vector, std::ofstream &out) +{ + for (size_t dim = 0; dim < (vector.size() - 1); ++dim) + out << vector[dim] << ", "; + out << vector[vector.size() - 1] << ")"; +} \ No newline at end of file diff --git a/src/LBM/LBM_BSP_Program.h b/src/LBM/LBM_BSP_Program.h new file mode 100644 index 0000000..04ccdcf --- /dev/null +++ b/src/LBM/LBM_BSP_Program.h @@ -0,0 +1,40 @@ +#ifndef INCLUDED_LBM_BSP_PROGRAM +#define INCLUDED_LBM_BSP_PROGRAM + +#include // size_t +#include +#include + +#include "Simulation.h" + +#include "node.h" +#include "parallel_bsp.h" + +class LBM_BSP_Program : public mcbsp::BSP_program { + + size_t p; + size_t s; + + size_t d_iterations; + std::vector d_domainSize; + + public: + + LBM_BSP_Program(size_t iterations, std::vector domainSize); + + protected: + + virtual void spmd(); + virtual BSP_program * newInstance(); + + private: + + LBM::Simulation * setup_simulation(); + void run_simulation(LBM::Simulation * sim); + void log_simulation_data(); + void show_vector(std::vector vector, std::ofstream &out); + +}; + + +#endif \ No newline at end of file diff --git a/src/LBM/Parallel/MCBSP.h b/src/LBM/Parallel/MCBSP.h new file mode 100644 index 0000000..e69de29 diff --git a/src/LBM/Simulation.cpp b/src/LBM/Simulation.cpp index 70fa7cc..dad0781 100644 --- a/src/LBM/Simulation.cpp +++ b/src/LBM/Simulation.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "../LBM/parallel_bsp.h" @@ -11,12 +12,12 @@ namespace LBM { Simulation::Simulation(Initializer_Ptr initializer) : d_domain(initializer->domain()) - { - // we'll send both the local idx and the direction which we we'll change - MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t[2]); - bsp_set_tagsize(&tag_size); - bsp_sync(); - } + {} + + Simulation::Simulation(std::unique_ptr domain) + : + d_domain(std::move(domain)) + {} Simulation::~Simulation() { @@ -48,6 +49,37 @@ namespace LBM { *nodes[idx].distributions[dir].neighbour = nodes[idx].distributions[dir].value; } + void Simulation::communicate(std::vector messengers) + { + for (auto messenger : messengers) + bsp_send(messenger.d_p, messenger.d_tag, &messenger.d_src, sizeof(double)); + bsp_sync(); + + MCBSP_NUMMSG_TYPE nmessages = 0; + MCBSP_BYTESIZE_TYPE nbytes = 0; + bsp_qsize(&nmessages, &nbytes); + for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) + { + size_t i[2]; + MCBSP_BYTESIZE_TYPE status; + bsp_get_tag(&status,&i); // i[0] = idx, i[1] = dir + if (status > 0) + { + double distribution = 0; + bsp_move(&distribution, sizeof(double)); + d_domain->nodes[i[0]].distributions[i[1]].nextValue = distribution; + } + } + // bsp_sync(); + } + + void Simulation::postStreamProcess() + { + for (size_t idx = 0; idx < d_domain->post_processors.size(); ++idx) + d_domain->post_processors[idx]->process(); + } + + void Simulation::collission(VelocitySet *set, std::vector &nodes) { double omega = d_domain->omega; @@ -71,42 +103,11 @@ namespace LBM { } } - void Simulation::postStreamProcess() - { - for (size_t idx = 0; idx < d_domain->post_processors.size(); ++idx) - d_domain->post_processors[idx]->process(); - } - - void Simulation::report(::Reporting::MatlabReporter reporter) { // reporter.reportOnTimeStep(d_domain->set, d_domain->nodes); } - void Simulation::communicate(std::vector messengers) - { - for (auto messenger : messengers) - bsp_send(messenger.d_p, messenger.d_tag, &messenger.d_src, sizeof(double)); - bsp_sync(); - - MCBSP_NUMMSG_TYPE nmessages = 0; - MCBSP_BYTESIZE_TYPE nbytes = 0; - bsp_qsize(&nmessages, &nbytes); - for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) - { - size_t i[2]; - MCBSP_BYTESIZE_TYPE status; - bsp_get_tag(&status,&i); // i[0] = idx, i[1] = dir - if (status > 0) - { - double distribution = 0; - bsp_move(&distribution, sizeof(double)); - d_domain->nodes[i[0]].distributions[i[1]].nextValue = distribution; - } - } - // bsp_sync(); - } - void Simulation::report() { size_t total_p = bsp_nprocs(); diff --git a/src/LBM/Simulation.h b/src/LBM/Simulation.h index ff7b614..f8370c7 100644 --- a/src/LBM/Simulation.h +++ b/src/LBM/Simulation.h @@ -20,7 +20,9 @@ namespace LBM { public: Simulation(Initializer_Ptr initializer); + Simulation(std::unique_ptr domain); ~Simulation(); + void step(); void report(::Reporting::MatlabReporter reporter); void report(); diff --git a/src/main.cpp b/src/main.cpp index f115c0d..d45657b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,103 +1,21 @@ #include "main.h" -#include -#include -#include -#include - -#include "Domains/DomainInitializer.h" -#include "Domains/BoxedDomain.h" -#include "Domains/LidDrivenCavity.h" -#include "Domains/PointDomain.h" -#include "VelocitySets/d2q9.h" -#include "LBM/node.h" -#include "LBM/parallel_bsp.h" - -#include - -using namespace Domains; -using std::make_unique; +#include +#include -// Global variables -size_t dx = 80; -size_t dy = 80; -size_t ITERATIONS; -size_t REPORT_PER_ITERATION = 10; -size_t P; +#include "LBM/LBM_BSP_Program.h" int main(int argc, char **argv) -try -{ - ITERATIONS = askForIterations(argc, argv); - P = askForProcessors(argc, argv); - - bsp_init(simulate, argc, argv); - simulate(); - - return 0; -} -catch(int x) -{ - return x; -} - -void simulate() { - bsp_begin(P); - - size_t p = bsp_nprocs(); - size_t s = bsp_pid(); - - std::vector domainSize{dx, dy}; + size_t iterations = askForIterations(argc, argv); + size_t processors = askForProcessors(argc, argv); - if (s == 0) - logSimulationData(domainSize); - - bsp_sync(); - double initialization_time = bsp_time(); - - D2Q9 set; - LBM::Simulation sim(make_unique(&set, domainSize, s, p)); - - // Log initialization time and prepare computation time - bsp_sync(); - double current_time = bsp_time(); - if (s == 0) - { - std::ofstream out("logs/timings.log", std::ios::out | std::ios::app); - // Initialization time - out << "IT: " << (current_time - initialization_time) << " sec, "; - } - double process_time = bsp_time(); + std::vector domain_size = {10, 10}; - run_simulation(sim); + LBM_BSP_Program lbm_program(iterations, domain_size); + lbm_program.begin(processors); - // Create a timestamp - bsp_sync(); - current_time = bsp_time(); - if (s == 0) - { - std::ofstream out("logs/timings.log", std::ios::out | std::ios::app); - // computation time - out << "CT: " << (current_time - process_time) << " sec" << '\n'; - } - - bsp_end(); -} - -/** - * Run ITERATIONS step and periodically report the current state of the simulation - * @param sim - */ -void run_simulation(LBM::Simulation &sim) -{ - for (size_t iter = 0; iter < ITERATIONS; ++iter) - { - sim.step(); - - if (iter % REPORT_PER_ITERATION == 0) - sim.report(); - } + return 0; } size_t askForIterations(int argc, char** argv) @@ -134,45 +52,3 @@ size_t askForProcessors(int argc, char** argv) } return P; } - -void logSimulationData(std::vector domainSize) -{ - std::ofstream out("logs/timings.log", std::ios::out | std::ios::app); - size_t p = bsp_nprocs(); - // Start by writing basic info to the file - // out << "LBM simulation using " << p << - // " processors to perform " << ITERATIONS << " iterations on the 'dummy' domain " << - // ", with set: " << "D2Q9" << " and domain size: ("; - out << "LBM, p: " << p << ", it: " << ITERATIONS << ", ds ("; - showVector(domainSize, out); - out << ", "; -} - -void showVector(std::vector vector, std::ofstream &out) -{ - for (size_t dim = 0; dim < (vector.size() - 1); ++dim) - out << vector[dim] << ", "; - out << vector[vector.size() - 1] << ")"; -} - -void createMatlabReport(LBM::Simulation &sim, size_t iter, std::vector domainSize) -{ - // Create a output file - std::ofstream out(createFileName(iter, "D2Q9", "PERIODIC", domainSize), std::ios::out | std::ios::app); - Reporting::MatlabReporter reporter(out); - sim.report(reporter); -} - -std::string createFileName(size_t iteration, std::string setName, std::string domainName, std::vector domainSize) -{ - // (iter, "D2Q9", "Lid_Driven_Cavity_5000_", domainSize); - // logs/D2Q9_Lid_Driven_Cavity_5000_50_100 - // logs/D2Q9_Boxed_10x10_100.txt - std::stringstream ss; - ss << "logs/" << setName << "_" << domainName << "_"; - for (size_t idx = 0; idx < domainSize.size() - 1; ++idx) - ss << domainSize[idx] << "x"; - ss << domainSize[domainSize.size() - 1]; - ss << "_" << ITERATIONS << "_" << iteration << ".txt"; - return ss.str(); -} \ No newline at end of file diff --git a/src/main.h b/src/main.h index 4e5f0fd..8405594 100644 --- a/src/main.h +++ b/src/main.h @@ -2,24 +2,11 @@ #define INCLUDED_MAIN #include -#include - -#include "LBM/Simulation.h" int main(int argc, char **argv); -void simulate(); -void run_simulation(LBM::Simulation &sim); // Setup functions size_t askForIterations(int argc, char** argv); size_t askForProcessors(int argc, char** argv); -// Logging functions -void logSimulationData(std::vector domainSize); -void showVector(std::vector vector, std::ofstream &out); - -// Reporting functions -void createMatlabReport(LBM::Simulation &sim, size_t iter, std::vector domainSize); -std::string createFileName(size_t iteration, std::string setName, std::string domainName, std::vector domainSize); - #endif \ No newline at end of file From bb983678dd82510b6352ccf337aa545393f7da3e Mon Sep 17 00:00:00 2001 From: Mark Redeman Date: Mon, 2 Nov 2015 00:37:13 +0100 Subject: [PATCH 3/4] Move retrieving of messengers in the DomainInitializer to a sperate function --- src/Domains/DomainInitializer.cpp | 51 ++++++++++++++++--------------- src/Domains/DomainInitializer.h | 3 +- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/Domains/DomainInitializer.cpp b/src/Domains/DomainInitializer.cpp index b5c2763..b0a47cd 100644 --- a/src/Domains/DomainInitializer.cpp +++ b/src/Domains/DomainInitializer.cpp @@ -16,9 +16,6 @@ namespace Domains { d_domain_size(domainSize) {} - DomainInitializer::~DomainInitializer() - {} - std::unique_ptr DomainInitializer::domain() { // setting the tagsize such that we can send the hash idx of a message @@ -39,28 +36,7 @@ namespace Domains { domain->set = d_set; domain->omega = omega(); - // setup messengers (for the parallelisation of the code) - bsp_sync(); - // get the destination from bsp and apply it to the appropriate messenger - MCBSP_NUMMSG_TYPE nmessages = 0; - MCBSP_BYTESIZE_TYPE nbytes = 0; - bsp_qsize(&nmessages, &nbytes); - for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) - { - size_t idx; // the hashIdx of the current messenger - MCBSP_BYTESIZE_TYPE status = 0; - bsp_get_tag(&status,&idx); - - if (status > 0) - { - size_t localIdx = 0; - bsp_move(&localIdx, status); - d_messengers[d_map_to_messenger[idx]].d_tag[0] = localIdx; - } - else - throw "Couldn't move the local idx during initialization phase."; - } - + retrieveMessengers(); domain->messengers = std::move(d_messengers); return domain; @@ -213,6 +189,31 @@ namespace Domains { // We don't need any post processors for this "dummy" domain } + void DomainInitializer::retrieveMessengers() + { + // setup messengers (for the parallelisation of the code) + bsp_sync(); + // get the destination from bsp and apply it to the appropriate messenger + MCBSP_NUMMSG_TYPE nmessages = 0; + MCBSP_BYTESIZE_TYPE nbytes = 0; + bsp_qsize(&nmessages, &nbytes); + for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) + { + size_t idx; // the hashIdx of the current messenger + MCBSP_BYTESIZE_TYPE status = 0; + bsp_get_tag(&status,&idx); + + if (status > 0) + { + size_t localIdx = 0; + bsp_move(&localIdx, status); + d_messengers[d_map_to_messenger[idx]].d_tag[0] = localIdx; + } + else + throw "Couldn't move the local idx during initialization phase."; + } + } + bool DomainInitializer::isInDomain(std::vector &position) { return true; diff --git a/src/Domains/DomainInitializer.h b/src/Domains/DomainInitializer.h index f5ee8ff..d8c0f0f 100644 --- a/src/Domains/DomainInitializer.h +++ b/src/Domains/DomainInitializer.h @@ -34,7 +34,7 @@ namespace Domains { public: DomainInitializer(VelocitySet *set, std::vector domainSize, size_t p = 0, size_t totalProcessors = 1); - virtual ~DomainInitializer(); + virtual ~DomainInitializer() = default; std::unique_ptr domain(); protected: @@ -46,6 +46,7 @@ namespace Domains { virtual bool isInDomain(std::vector &position); virtual void createPostProcessors(std::vector &nodes); + virtual void retrieveMessengers(); virtual double omega(); From 4c64d6215fb4b2e6bd2ce2ef9a6331e3cac64f3f Mon Sep 17 00:00:00 2001 From: Mark Redeman Date: Tue, 3 Nov 2015 12:38:58 +0100 Subject: [PATCH 4/4] Sepperate the bsp specific functions into their own files. This will hopefully make it easier to switch to different parallel / communication methods. --- src/Domains/DomainInitializer.cpp | 16 ++++---- src/LBM/LBM_BSP_Program.cpp | 5 +-- src/LBM/Simulation.cpp | 66 ------------------------------ src/LBM/Simulation_Communicate.cpp | 48 ++++++++++++++++++++++ src/LBM/Simulation_Report.cpp | 64 +++++++++++++++++++++++++++++ src/main.cpp | 5 --- 6 files changed, 123 insertions(+), 81 deletions(-) create mode 100644 src/LBM/Simulation_Communicate.cpp create mode 100644 src/LBM/Simulation_Report.cpp diff --git a/src/Domains/DomainInitializer.cpp b/src/Domains/DomainInitializer.cpp index b0a47cd..41e3d48 100644 --- a/src/Domains/DomainInitializer.cpp +++ b/src/Domains/DomainInitializer.cpp @@ -18,11 +18,6 @@ namespace Domains { std::unique_ptr DomainInitializer::domain() { - // setting the tagsize such that we can send the hash idx of a message - MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t); - bsp_set_tagsize(&tag_size); - bsp_sync(); - createNodes(); std::unique_ptr domain(new Domain); @@ -35,8 +30,6 @@ namespace Domains { domain->set = d_set; domain->omega = omega(); - - retrieveMessengers(); domain->messengers = std::move(d_messengers); return domain; @@ -76,6 +69,13 @@ namespace Domains { } } + // Connecting the nodes to their neighbours and set up the messengers + + // setting the tagsize such that we can send the hash idx of a message + MCBSP_BYTESIZE_TYPE tag_size = sizeof(size_t); + bsp_set_tagsize(&tag_size); + bsp_sync(); + for (size_t idx = 0; idx < d_nodes.size(); ++idx) connectNodeToNeighbours(idx); @@ -85,6 +85,8 @@ namespace Domains { size_t node_idx = d_messengers[idx].d_tag[0]; d_nodes[node_idx].distributions[d_messengers[idx].d_tag[1]].neighbour = &d_messengers[idx].d_src; } + + retrieveMessengers(); } Node DomainInitializer::initializeNodeAt(std::vector &position) diff --git a/src/LBM/LBM_BSP_Program.cpp b/src/LBM/LBM_BSP_Program.cpp index f362be6..6f2a2be 100644 --- a/src/LBM/LBM_BSP_Program.cpp +++ b/src/LBM/LBM_BSP_Program.cpp @@ -24,9 +24,8 @@ void LBM_BSP_Program::spmd() { p = bsp_nprocs(); s = bsp_pid(); - auto sim = setup_simulation(); - run_simulation(sim); - delete sim; + std::unique_ptr sim(setup_simulation()); + run_simulation(sim.get()); } diff --git a/src/LBM/Simulation.cpp b/src/LBM/Simulation.cpp index dad0781..285d883 100644 --- a/src/LBM/Simulation.cpp +++ b/src/LBM/Simulation.cpp @@ -1,12 +1,7 @@ #include "Simulation.h" #include "node.h" -#include -#include -#include #include -#include "../LBM/parallel_bsp.h" - namespace LBM { Simulation::Simulation(Initializer_Ptr initializer) @@ -49,37 +44,12 @@ namespace LBM { *nodes[idx].distributions[dir].neighbour = nodes[idx].distributions[dir].value; } - void Simulation::communicate(std::vector messengers) - { - for (auto messenger : messengers) - bsp_send(messenger.d_p, messenger.d_tag, &messenger.d_src, sizeof(double)); - bsp_sync(); - - MCBSP_NUMMSG_TYPE nmessages = 0; - MCBSP_BYTESIZE_TYPE nbytes = 0; - bsp_qsize(&nmessages, &nbytes); - for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) - { - size_t i[2]; - MCBSP_BYTESIZE_TYPE status; - bsp_get_tag(&status,&i); // i[0] = idx, i[1] = dir - if (status > 0) - { - double distribution = 0; - bsp_move(&distribution, sizeof(double)); - d_domain->nodes[i[0]].distributions[i[1]].nextValue = distribution; - } - } - // bsp_sync(); - } - void Simulation::postStreamProcess() { for (size_t idx = 0; idx < d_domain->post_processors.size(); ++idx) d_domain->post_processors[idx]->process(); } - void Simulation::collission(VelocitySet *set, std::vector &nodes) { double omega = d_domain->omega; @@ -102,40 +72,4 @@ namespace LBM { delete[] node_equilibrium; } } - - void Simulation::report(::Reporting::MatlabReporter reporter) - { - // reporter.reportOnTimeStep(d_domain->set, d_domain->nodes); - } - - void Simulation::report() - { - size_t total_p = bsp_nprocs(); - size_t s = bsp_pid(); - double *densities = new double[total_p](); - double current_density = 0; - bsp_push_reg(densities,total_p * sizeof(double)); - bsp_sync(); - - for (auto node : d_domain->nodes) - current_density += density(d_domain->set, node); - - // send density to each processor - for (size_t t = 0; t < total_p; t++) - bsp_put(t, ¤t_density, densities, s * sizeof(double), sizeof(double)); - - bsp_sync(); - // now calculate the total density - double total_density = 0; - for (size_t t = 0; t < total_p; t++) - total_density += densities[t]; - - bsp_pop_reg(densities); - - if (s == 0) - std::cout << "Total density: " << total_density << '\n'; - - delete[] densities; - } - } \ No newline at end of file diff --git a/src/LBM/Simulation_Communicate.cpp b/src/LBM/Simulation_Communicate.cpp new file mode 100644 index 0000000..49e2013 --- /dev/null +++ b/src/LBM/Simulation_Communicate.cpp @@ -0,0 +1,48 @@ +#include "Simulation.h" +#include "node.h" +#include + +#include "../LBM/parallel_bsp.h" + +namespace LBM { + + // The communication and stream step use a push method + void Simulation::communicate(std::vector messengers) + { + // If a node's neighbor is not present on the current processor (P_n) then we + // will want to send the distribution that processor P_n needs + // + // each messenger has a double d_src which contains the value of the distribution + // that needs to be send to P_n + // This distribution has been set during the streaming phase + // + // We will sent both the distribution (d_src) and a tag that contains the local index of + // the node and the index of the direction from which the distribution is coming from + + // First we will send the distribution of each messenger to the corresponding processor + // we also send the node index and the direction index to the processor by using d_tag + for (auto messenger : messengers) + bsp_send(messenger.d_p, messenger.d_tag, &messenger.d_src, sizeof(double)); + bsp_sync(); + + // Next we retrieve the distributions that have been send to this processor + // first we determine the target of the distribution (which is stored in the size_t target[2]) + // next we move the distribution to this target's next value + MCBSP_NUMMSG_TYPE nmessages = 0; + MCBSP_BYTESIZE_TYPE nbytes = 0; + bsp_qsize(&nmessages, &nbytes); + for (MCBSP_NUMMSG_TYPE n = 0; n < nmessages; ++n) + { + size_t target[2]; + MCBSP_BYTESIZE_TYPE status; + bsp_get_tag(&status,&target); // i[0] = idx, i[1] = dir + if (status > 0) + { + double distribution = 0; + bsp_move(&distribution, sizeof(double)); + d_domain->nodes[target[0]].distributions[target[1]].nextValue = distribution; + } + } + } + +} \ No newline at end of file diff --git a/src/LBM/Simulation_Report.cpp b/src/LBM/Simulation_Report.cpp new file mode 100644 index 0000000..6adc826 --- /dev/null +++ b/src/LBM/Simulation_Report.cpp @@ -0,0 +1,64 @@ +#include "Simulation.h" +#include "node.h" +#include +#include +#include +#include +#include + +#include "../Reporting/MatlabReporter.h" + +#include "../LBM/parallel_bsp.h" + +namespace LBM { + + void Simulation::report(::Reporting::MatlabReporter reporter) + { + // reporter.reportOnTimeStep(d_domain->set, d_domain->nodes); + } + + void Simulation::report() + { + + for (size_t p = 0; p < bsp_nprocs(); ++p) + { + if (p == bsp_pid()) + { + std::ofstream out("logs/test.txt", std::ios::out | std::ios::app); + Reporting::MatlabReporter reporter(out); + reporter.reportOnTimeStep(d_domain->set, d_domain->nodes); + + } + bsp_sync(); + } + + + size_t total_p = bsp_nprocs(); + size_t s = bsp_pid(); + double *densities = new double[total_p](); + double current_density = 0; + bsp_push_reg(densities,total_p * sizeof(double)); + bsp_sync(); + + for (auto node : d_domain->nodes) + current_density += density(d_domain->set, node); + + // send density to each processor + for (size_t t = 0; t < total_p; t++) + bsp_put(t, ¤t_density, densities, s * sizeof(double), sizeof(double)); + + bsp_sync(); + // now calculate the total density + double total_density = 0; + for (size_t t = 0; t < total_p; t++) + total_density += densities[t]; + + bsp_pop_reg(densities); + + if (s == 0) + std::cout << "Total density: " << total_density << '\n'; + + delete[] densities; + } + +} \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index d45657b..31dcc56 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -45,10 +45,5 @@ size_t askForProcessors(int argc, char** argv) std::cin >> P; } - if (P > bsp_nprocs()) - { - std::cout << "Sorry, not enough processors available." << std::endl; - exit(0); - } return P; }