PetsC API Changes and Fixes

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.18.4)
 include(config/petscCompilers.cmake)
 
 # Set the project details
-project(ablateLibrary VERSION 0.12.36)
+project(ablateLibrary VERSION 0.13.00)
 
 # Load the Required 3rd Party Libaries
 pkg_check_modules(PETSc REQUIRED IMPORTED_TARGET GLOBAL PETSc)

CMakePresets.json

@@ -34,6 +34,36 @@
         "PKG_CONFIG_PATH": "$env{PETSC_DIR}/arch-ablate-opt/lib/pkgconfig:$penv{PKG_CONFIG_PATH}"
       }
     },
+    {
+      "name": "OptMainPetsc",
+      "displayName": "CLion Opt Config",
+      "description": "Default build for ABLATE in CLion",
+      "binaryDir": "${sourceDir}/cmake-build-MainPetsc",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "COMPILE_MPI_COMMAND": "$env{PETSC_DIR}/arch-ablate-opt/bin/mpirun"
+      },
+      "environment": {
+        "PETSC_DIR": "/home/klbud/Software/PetscUpdating/petsc",
+        "PETSC_ARCH": "arch-ablate-opt",
+        "PKG_CONFIG_PATH": "$env{PETSC_DIR}/arch-ablate-opt/lib/pkgconfig:$penv{PKG_CONFIG_PATH}"
+      }
+    },
+    {
+      "name": "OptFluentPetsc",
+      "displayName": "CLion Opt Config",
+      "description": "Default build for ABLATE in CLion",
+      "binaryDir": "${sourceDir}/cmake-build-FluentPetsc",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "COMPILE_MPI_COMMAND": "$env{PETSC_DIR}/arch-ablate-opt/bin/mpirun"
+      },
+      "environment": {
+        "PETSC_DIR": "/home/klbud/Software/PetscUpdating/petscW",
+        "PETSC_ARCH": "arch-ablate-opt",
+        "PKG_CONFIG_PATH": "$env{PETSC_DIR}/arch-ablate-opt/lib/pkgconfig:$penv{PKG_CONFIG_PATH}"
+      }
+    },
     {
       "name": "local-ablate-opt-info",
       "displayName": "CLion RelWithDebugInfo Config",

src/boundarySolver/physics/subModels/oneDimensionHeatTransfer.cpp

@@ -95,7 +95,7 @@ ablate::boundarySolver::physics::subModels::OneDimensionHeatTransfer::OneDimensi
 
         // get the global section
         PetscSection section;
-        DMGetSection(subModelDm, &section) >> utilities::PetscUtilities::checkError;
+        DMGetLocalSection(subModelDm, &section) >> utilities::PetscUtilities::checkError;
 
         // Determine the BC Node
         for (PetscInt p = pStart; p < pEnd; ++p) {

src/domain/cadFile.cpp

@@ -60,7 +60,7 @@ DM ablate::domain::CadFile::ReadDMFromCadFile(const std::string& name, const std
     DMPlexSetRefinementUniform(dm, PETSC_TRUE) >> utilities::PetscUtilities::checkError;
 
     // inflate the mesh
-    DMPlexInflateToGeomModel(dm) >> utilities::PetscUtilities::checkError;
+    DMPlexInflateToGeomModel(dm, PETSC_TRUE) >> utilities::PetscUtilities::checkError;
     return dm;
 }
 

src/domain/domain.cpp

@@ -293,7 +293,7 @@ bool ablate::domain::Domain::CheckFieldValues(Vec globSourceVector) {
 
         // get the global section
         PetscSection sourceSection;
-        DMGetSection(sourceDM, &sourceSection) >> utilities::PetscUtilities::checkError;
+        DMGetLocalSection(sourceDM, &sourceSection) >> utilities::PetscUtilities::checkError;
 
         for (PetscInt p = pStart; p < pEnd; ++p) {
             const PetscScalar* sourceAtP = nullptr;

src/particles/initializers/boxInitializer.cpp

@@ -28,9 +28,12 @@ void ablate::particles::initializers::BoxInitializer::Initialize(ablate::domain:
     DMGetDimension(particleDm, &dim) >> utilities::PetscUtilities::checkError;
 
     PetscInt Np = 1;
-    PetscInt *cellid, n[3];
+    PetscInt Nfc;
+    DMSwarmCellDM cellDm;
+    PetscInt *swarm_cellid, n[3];
     PetscReal x[3], dx[3];
     PetscScalar *coords;
+    const char **coordFields, *cellid;
     PetscMPIInt rank;
 
     DMSetFromOptions(particleDm) >> utilities::PetscUtilities::checkError;
@@ -49,7 +52,12 @@ void ablate::particles::initializers::BoxInitializer::Initialize(ablate::domain:
 
     DMSwarmSetLocalSizes(particleDm, Np, 0) >> utilities::PetscUtilities::checkError;
     DMSetFromOptions(particleDm) >> utilities::PetscUtilities::checkError;
-    DMSwarmGetField(particleDm, DMSwarmPICField_coor, nullptr, nullptr, (void **)&coords) >> utilities::PetscUtilities::checkError;
+
+    DMSwarmGetCellDMActive(particleDm, &cellDm) >> utilities::PetscUtilities::checkError;
+    DMSwarmCellDMGetCellID(cellDm, &cellid) >> utilities::PetscUtilities::checkError;
+    DMSwarmCellDMGetCoordinateFields(cellDm, &Nfc, &coordFields);
+    DMSwarmGetField(particleDm, coordFields[0], NULL, NULL, (void **)&coords) >> utilities::PetscUtilities::checkError;
+
     if (rank == 0) {
         switch (dim) {
             case 2:
@@ -83,12 +91,13 @@ void ablate::particles::initializers::BoxInitializer::Initialize(ablate::domain:
                 throw std::runtime_error("Do not support particle layout in dimension " + std::to_string(dim));
         }
     }
-    DMSwarmRestoreField(particleDm, DMSwarmPICField_coor, nullptr, nullptr, (void **)&coords) >> utilities::PetscUtilities::checkError;
-    DMSwarmGetField(particleDm, DMSwarmPICField_cellid, nullptr, nullptr, (void **)&cellid) >> utilities::PetscUtilities::checkError;
+    DMSwarmRestoreField(particleDm, coordFields[0], NULL, NULL, (void **)&coords) >> utilities::PetscUtilities::checkError;
+
+    DMSwarmGetField(particleDm, cellid, NULL, NULL, (void **)&swarm_cellid) >> utilities::PetscUtilities::checkError;
     for (PetscInt p = 0; p < Np; ++p) {
-        cellid[p] = 0;
+        swarm_cellid[p] = 0;
     }
-    DMSwarmRestoreField(particleDm, DMSwarmPICField_cellid, nullptr, nullptr, (void **)&cellid) >> utilities::PetscUtilities::checkError;
+    DMSwarmRestoreField(particleDm, cellid, NULL, NULL, (void **)&swarm_cellid) >> utilities::PetscUtilities::checkError;
     DMSwarmMigrate(particleDm, PETSC_TRUE) >> utilities::PetscUtilities::checkError;
 }
 

src/particles/initializers/cellInitializer.cpp

@@ -11,16 +11,20 @@ void ablate::particles::initializers::CellInitializer::Initialize(ablate::domain
     DMSwarmSetLocalSizes(particleDm, (cEnd - cStart) * particlesPerCellLocal, 0) >> utilities::PetscUtilities::checkError;
 
     // set the cell ids
-    PetscInt *cellid;
-    DMSwarmGetField(particleDm, DMSwarmPICField_cellid, nullptr, nullptr, (void **)&cellid) >> utilities::PetscUtilities::checkError;
+    DMSwarmCellDM celldm;
+    const char *cellid;
+    PetscInt *swarm_cellid;
+    DMSwarmGetCellDMActive(particleDm, &celldm) >> utilities::PetscUtilities::checkError;
+    DMSwarmCellDMGetCellID(celldm, &cellid) >> utilities::PetscUtilities::checkError;
+
+    DMSwarmGetField(particleDm, cellid, NULL, NULL, (void **)&swarm_cellid) >> utilities::PetscUtilities::checkError;
     for (PetscInt c = cStart; c < cEnd; ++c) {
         for (PetscInt p = 0; p < particlesPerCellLocal; ++p) {
             const PetscInt n = c * particlesPerCellLocal + p;
-            cellid[n] = c;
+            swarm_cellid[n] = c;
         }
     }
-
-    DMSwarmRestoreField(particleDm, DMSwarmPICField_cellid, nullptr, nullptr, (void **)&cellid) >> utilities::PetscUtilities::checkError;
+    DMSwarmRestoreField(particleDm, cellid, nullptr, nullptr, (void **)&swarm_cellid) >> utilities::PetscUtilities::checkError;
     DMSwarmSetPointCoordinatesRandom(particleDm, particlesPerCellLocal) >> utilities::PetscUtilities::checkError;
 }
 

src/particles/particleSolver.cpp

@@ -115,12 +115,12 @@ void ablate::particles::ParticleSolver::Setup() {
 }
 
 void ablate::particles::ParticleSolver::Initialize() {
-    // before setting up the flow finalize the fields
-    DMSwarmFinalizeFieldRegister(swarmDm) >> utilities::PetscUtilities::checkError;
-
     // associate the swarm with the cell dm
     DMSwarmSetCellDM(swarmDm, subDomain->GetDM()) >> utilities::PetscUtilities::checkError;
 
+    // before setting up the flow finalize the fields
+    DMSwarmFinalizeFieldRegister(swarmDm) >> utilities::PetscUtilities::checkError;
+
     // name the particle domain
     PetscObjectSetOptions((PetscObject)swarmDm, petscOptions) >> utilities::PetscUtilities::checkError;
     PetscObjectSetName((PetscObject)swarmDm, GetSolverId().c_str()) >> utilities::PetscUtilities::checkError;

src/radiation/orthogonalRadiation.cpp

@@ -63,14 +63,22 @@ void ablate::radiation::OrthogonalRadiation::Setup(const ablate::domain::Range&
         utilities::PetscUtilities::checkError;  //!< Set the number of initial particles to the number of rays in the subdomain. Set the buffer size to zero.
 
     /** Declare some information associated with the field declarations */
+    DMSwarmCellDM cellDm;               //!< Swarm cell DM
     PetscReal* coord;                   //!< Pointer to the coordinate field information
-    PetscInt* index;                    //!< Pointer to the cell index information
+    const char *cellid, **coordFields;  //!< Swarm cellId, and coordinate fields
+    PetscInt Nfc;                       //!< Number of coordinate fields
+    PetscInt* swarm_index;              //!< Pointer to the cell index information
     struct Virtualcoord* virtualcoord;  //!< Pointer to the primary (virtual) coordinate field information
     struct Identifier* identifier;      //!< Pointer to the ray identifier information
 
+    /** Get the swarm cell DM, cell Id and coordinate fields */
+    DMSwarmGetCellDMActive(radSearch, &cellDm) >> utilities::PetscUtilities::checkError;
+    DMSwarmCellDMGetCellID(cellDm, &cellid) >> utilities::PetscUtilities::checkError;
+    DMSwarmCellDMGetCoordinateFields(cellDm, &Nfc, &coordFields);
+
     /** Get the fields associated with the particle swarm so that they can be modified */
-    DMSwarmGetField(radSearch, DMSwarmPICField_coor, nullptr, nullptr, (void**)&coord) >> utilities::PetscUtilities::checkError;
-    DMSwarmGetField(radSearch, DMSwarmPICField_cellid, nullptr, nullptr, (void**)&index) >> utilities::PetscUtilities::checkError;
+    DMSwarmGetField(radSearch, coordFields[0], nullptr, nullptr, (void**)&coord) >> utilities::PetscUtilities::checkError;
+    DMSwarmGetField(radSearch, cellid, nullptr, nullptr, (void**)&swarm_index) >> utilities::PetscUtilities::checkError;
     DMSwarmGetField(radSearch, IdentifierField, nullptr, nullptr, (void**)&identifier) >> utilities::PetscUtilities::checkError;
     DMSwarmGetField(radSearch, VirtualCoordField, nullptr, nullptr, (void**)&virtualcoord) >> utilities::PetscUtilities::checkError;
 
@@ -125,8 +133,8 @@ void ablate::radiation::OrthogonalRadiation::Setup(const ablate::domain::Range&
     }
 
     /** Restore the fields associated with the particles */
-    DMSwarmRestoreField(radSearch, DMSwarmPICField_coor, nullptr, nullptr, (void**)&coord) >> utilities::PetscUtilities::checkError;
-    DMSwarmRestoreField(radSearch, DMSwarmPICField_cellid, nullptr, nullptr, (void**)&index) >> utilities::PetscUtilities::checkError;
+    DMSwarmRestoreField(radSearch, coordFields[0], nullptr, nullptr, (void**)&coord) >> utilities::PetscUtilities::checkError;
+    DMSwarmRestoreField(radSearch, cellid, nullptr, nullptr, (void**)&swarm_index) >> utilities::PetscUtilities::checkError;
     DMSwarmRestoreField(radSearch, IdentifierField, nullptr, nullptr, (void**)&identifier) >> utilities::PetscUtilities::checkError;
     DMSwarmRestoreField(radSearch, VirtualCoordField, nullptr, nullptr, (void**)&virtualcoord) >> utilities::PetscUtilities::checkError;